{ "schema_version": 1, "dataset_git_commit": "70a514f06f9e156be480e2a0b28a1d3380467ace", "dataset_fingerprint": "fc531bdb2e7c9d10e2074519bd3e0d0f66bc8ec970d20f25cdb628054784c6de", "expected_task_count": 292, "run_id": "qualitative-20260917", "mock": false, "grading_version": "1", "release": "0.3.0", "evaluation_protocol": "deed0698dca65b5e24f3b06766d2d5344bbf0964e871167cebfc5e4aaba554fa", "finalized_at": "2026-09-17T12:10:52.016183+00:00", "source_updated_at": "2026-09-17T12:10:06.516568+00:00", "source_git_commit": "bd677173c19051bb9316d2d7075d2a931b8bacc3", "finalizer_git_commit": "bd677173c19051bb9316d2d7075d2a931b8bacc3", "finalizer_git_dirty": false, "artifact_type": "layoutbench-final-results", "publication_status": "finalized", "benchmark_id": "layoutbench", "version": "0.3.0", "status": "pilot", "human_validation": "not_performed", "numeric_scoring": { "distance": "absolute-px-error", "score_ceiling": 16, "score_formula": "100 * (1 - min(err_px / 16, 1))", "tight_ceiling": 4, "tight_formula": "100 * (1 - min(err_px / 4, 1))", "exact_bands": [ 1, 2, 4, 8 ], "interpretation": "Engineering normalization; not a just-noticeable-difference threshold." }, "dataset_manifest": "dataset/layoutbench-v0.3/manifest.json", "dataset_path": "dataset/layoutbench-v0.3", "comparison": { "scope": "full", "count": 292, "task_ids": [ "layoutbench-blockalign-001", "layoutbench-blockalign-002", "layoutbench-blockalign-003", "layoutbench-blockalign-004", "layoutbench-blockalign-005", "layoutbench-blockalign-006", "layoutbench-blockalign-007", "layoutbench-blockalign-008", "layoutbench-blockalign-009", "layoutbench-blockalign-010", "layoutbench-blockalign-011", "layoutbench-blockalign-012", "layoutbench-crossalign-001", "layoutbench-crossalign-002", "layoutbench-crossalign-003", "layoutbench-crossalign-004", "layoutbench-crossalign-005", "layoutbench-crossalign-006", "layoutbench-crossalign-007", "layoutbench-crossalign-008", "layoutbench-crossalign-009", "layoutbench-crossalign-010", "layoutbench-crossalign-011", "layoutbench-crossalign-012", "layoutbench-crossalign-013", "layoutbench-crossalign-014", "layoutbench-crossalign-015", "layoutbench-crossalign-016", "layoutbench-direction-001", "layoutbench-direction-002", "layoutbench-direction-003", "layoutbench-direction-004", "layoutbench-direction-005", "layoutbench-direction-006", "layoutbench-direction-007", "layoutbench-direction-008", "layoutbench-distribution-001", "layoutbench-distribution-002", "layoutbench-distribution-003", "layoutbench-distribution-004", "layoutbench-distribution-005", "layoutbench-distribution-006", "layoutbench-distribution-007", "layoutbench-distribution-008", "layoutbench-distribution-009", "layoutbench-distribution-010", "layoutbench-distribution-011", "layoutbench-distribution-012", "layoutbench-distribution-013", "layoutbench-distribution-014", "layoutbench-distribution-015", "layoutbench-distribution-016", "layoutbench-distribution-017", "layoutbench-distribution-018", "layoutbench-distribution-019", "layoutbench-distribution-020", "layoutbench-distribution-021", "layoutbench-distribution-022", "layoutbench-distribution-023", "layoutbench-distribution-024", "layoutbench-gapcompare-001", "layoutbench-gapcompare-002", "layoutbench-gapcompare-003", "layoutbench-gapcompare-004", "layoutbench-gapcompare-005", "layoutbench-gapcompare-006", "layoutbench-gapcompare-007", "layoutbench-gapcompare-008", "layoutbench-gapcompare-009", "layoutbench-gapcompare-010", "layoutbench-gapcompare-011", "layoutbench-gapcompare-012", "layoutbench-gridcols-001", "layoutbench-gridcols-002", "layoutbench-gridcols-003", "layoutbench-gridcols-004", "layoutbench-gridcols-005", "layoutbench-gridcols-006", "layoutbench-gridcols-007", "layoutbench-gridcols-008", "layoutbench-gridcols-009", "layoutbench-gridcols-010", "layoutbench-gridcols-011", "layoutbench-gridcols-012", "layoutbench-gridgaps-001", "layoutbench-gridgaps-002", "layoutbench-gridgaps-003", "layoutbench-gridgaps-004", "layoutbench-gridgaps-005", "layoutbench-gridgaps-006", "layoutbench-gridgaps-007", "layoutbench-gridgaps-008", "layoutbench-gridgaps-009", "layoutbench-gridgaps-010", "layoutbench-gridgaps-011", "layoutbench-gridgaps-012", "layoutbench-gridrows-001", "layoutbench-gridrows-002", "layoutbench-gridrows-003", "layoutbench-gridrows-004", "layoutbench-gridrows-005", "layoutbench-gridrows-006", "layoutbench-gridrows-007", "layoutbench-gridrows-008", "layoutbench-gridrows-009", "layoutbench-gridrows-010", "layoutbench-gridrows-011", "layoutbench-gridrows-012", "layoutbench-gridspan-001", "layoutbench-gridspan-002", "layoutbench-gridspan-003", "layoutbench-gridspan-004", "layoutbench-gridspan-005", "layoutbench-gridspan-006", "layoutbench-gridspan-007", "layoutbench-gridspan-008", "layoutbench-gridspan-009", "layoutbench-gridspan-010", "layoutbench-gridspan-011", "layoutbench-gridspan-012", "layoutbench-gridtracks-001", "layoutbench-gridtracks-002", "layoutbench-gridtracks-003", "layoutbench-gridtracks-004", "layoutbench-gridtracks-005", "layoutbench-gridtracks-006", "layoutbench-gridtracks-007", "layoutbench-gridtracks-008", "layoutbench-gridtracks-009", "layoutbench-gridtracks-010", "layoutbench-gridtracks-011", "layoutbench-gridtracks-012", "layoutbench-groupgap-001", "layoutbench-groupgap-002", "layoutbench-groupgap-003", "layoutbench-groupgap-004", "layoutbench-groupgap-005", "layoutbench-groupgap-006", "layoutbench-groupgap-007", "layoutbench-groupgap-008", "layoutbench-groupgap-009", "layoutbench-groupgap-010", "layoutbench-groupgap-011", "layoutbench-groupgap-012", "layoutbench-nestedflow-001", "layoutbench-nestedflow-002", "layoutbench-nestedflow-003", "layoutbench-nestedflow-004", "layoutbench-nestedflow-005", "layoutbench-nestedflow-006", "layoutbench-nestedflow-007", "layoutbench-nestedflow-008", "layoutbench-nestedflow-009", "layoutbench-nestedflow-010", "layoutbench-nestedflow-011", "layoutbench-nestedflow-012", "layoutbench-nestedflow-013", "layoutbench-nestedflow-014", "layoutbench-nestedflow-015", "layoutbench-nestedflow-016", "layoutbench-nestedflow-017", "layoutbench-nestedflow-018", "layoutbench-nestedflow-019", "layoutbench-nestedflow-020", "layoutbench-nestedflow-021", "layoutbench-nestedflow-022", "layoutbench-nestedflow-023", "layoutbench-nestedflow-024", "layoutbench-nestedwrap-001", "layoutbench-nestedwrap-002", "layoutbench-nestedwrap-003", "layoutbench-nestedwrap-004", "layoutbench-nestedwrap-005", "layoutbench-nestedwrap-006", "layoutbench-nestedwrap-007", "layoutbench-nestedwrap-008", "layoutbench-nestedwrap-009", "layoutbench-nestedwrap-010", "layoutbench-nestedwrap-011", "layoutbench-nestedwrap-012", "layoutbench-nestedwrap-013", "layoutbench-nestedwrap-014", "layoutbench-nestedwrap-015", "layoutbench-nestedwrap-016", "layoutbench-nestedwrap-017", "layoutbench-nestedwrap-018", "layoutbench-nestedwrap-019", "layoutbench-nestedwrap-020", "layoutbench-nestedwrap-021", "layoutbench-nestedwrap-022", "layoutbench-nestedwrap-023", "layoutbench-nestedwrap-024", "layoutbench-padcompare-001", "layoutbench-padcompare-002", "layoutbench-padcompare-003", "layoutbench-padcompare-004", "layoutbench-padcompare-005", "layoutbench-padcompare-006", "layoutbench-padcompare-007", "layoutbench-padcompare-008", "layoutbench-padcompare-009", "layoutbench-padcompare-010", "layoutbench-padcompare-011", "layoutbench-padcompare-012", "layoutbench-textalign-001", "layoutbench-textalign-002", "layoutbench-textalign-003", "layoutbench-textalign-004", "layoutbench-textalign-005", "layoutbench-textalign-006", "layoutbench-textalign-007", "layoutbench-textalign-008", "layoutbench-textalign-009", "layoutbench-textalign-010", "layoutbench-textalign-011", "layoutbench-textalign-012", "layoutbench-textalign-013", "layoutbench-textalign-014", "layoutbench-textalign-015", "layoutbench-textalign-016", "layoutbench-textcolumns-001", "layoutbench-textcolumns-002", "layoutbench-textcolumns-003", "layoutbench-textcolumns-004", "layoutbench-textcolumns-005", "layoutbench-textcolumns-006", "layoutbench-textcolumns-007", "layoutbench-textcolumns-008", "layoutbench-textcolumns-009", "layoutbench-textcolumns-010", "layoutbench-textcolumns-011", "layoutbench-textcolumns-012", "layoutbench-topflow-001", "layoutbench-topflow-002", "layoutbench-topflow-003", "layoutbench-topflow-004", "layoutbench-topflow-005", "layoutbench-topflow-006", "layoutbench-topflow-007", "layoutbench-topflow-008", "layoutbench-topflow-009", "layoutbench-topflow-010", "layoutbench-topflow-011", "layoutbench-topflow-012", "layoutbench-topflow-013", "layoutbench-topflow-014", "layoutbench-topflow-015", "layoutbench-topflow-016", "layoutbench-topflow-017", "layoutbench-topflow-018", "layoutbench-topflow-019", "layoutbench-topflow-020", "layoutbench-topflow-021", "layoutbench-topflow-022", "layoutbench-topflow-023", "layoutbench-topflow-024", "layoutbench-widthcompare-001", "layoutbench-widthcompare-002", "layoutbench-widthcompare-003", "layoutbench-widthcompare-004", "layoutbench-widthcompare-005", "layoutbench-widthcompare-006", "layoutbench-widthcompare-007", "layoutbench-widthcompare-008", "layoutbench-widthcompare-009", "layoutbench-widthcompare-010", "layoutbench-widthcompare-011", "layoutbench-widthcompare-012", "layoutbench-wrapalign-001", "layoutbench-wrapalign-002", "layoutbench-wrapalign-003", "layoutbench-wrapalign-004", "layoutbench-wrapalign-005", "layoutbench-wrapalign-006", "layoutbench-wrapalign-007", "layoutbench-wrapalign-008", "layoutbench-wrapalign-009", "layoutbench-wrapalign-010", "layoutbench-wrapalign-011", "layoutbench-wrapalign-012", "layoutbench-wrapcount-001", "layoutbench-wrapcount-002", "layoutbench-wrapcount-003", "layoutbench-wrapcount-004", "layoutbench-wrapcount-005", "layoutbench-wrapcount-006", "layoutbench-wrapcount-007", "layoutbench-wrapcount-008", "layoutbench-wrapcount-009", "layoutbench-wrapcount-010", "layoutbench-wrapcount-011", "layoutbench-wrapcount-012" ], "cohort_fingerprint": "56e2fb761613c5caa49412d9ea2c156fe82287109203dd73d974c23290fb64da", "model_ids": [ "claude-fable-5-1", "claude-haiku-4-5-20251001", "claude-opus-5", "claude-sonnet-5", "gemini-3.1-pro-preview", "gemini-3.5-flash-lite", "gemini-3.8-flash", "gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-6-astra", "muse-spark-1.2", "muse-spark-1.3" ] }, "cost_basis": "Recorded metered token usage multiplied by recorded model pricing per scored input; excludes unmetered reserves and separate infrastructure attempts.", "statistical_scope": "Descriptive finite pilot. Related images share groups; no independent-sample confidence intervals or combined cross-family score.", "campaign": { "final_response_count": 3796, "attempt_count": 3799, "infrastructure_attempt_count": 3, "infrastructure_error_kinds": { "credits": 3 }, "unresolved_infrastructure_task_count": 0, "spent_cost_usd": 34.56476569999998, "budget_usd": 100.0, "cost_incomplete": false, "estimated_cost_attempt_count": 6 }, "models": [ { "model_id": "claude-fable-5-1", "model_config": { "api_key_env": "ANTHROPIC_API_KEY", "base_url": null, "display_name": "Claude Fable 5.1", "enabled": true, "id": "claude-fable-5-1", "input_per_m": 10.0, "max_output_tokens": 4096, "model": "claude-fable-5-1", "notes": "Latest Fable. Adaptive thinking is always on; default effort is high.", "output_per_m": 50.0, "pricing_notes": "Standard online API rates in USD per million tokens; excludes caching, batch discounts and tools.", "pricing_source_url": "https://platform.claude.com/docs/en/about-claude/pricing", "provider": "anthropic", "source_url": "https://platform.claude.com/docs/en/models/overview" }, "model_config_fingerprint": "5651e81821421e372074e9b58db5cced4e5fb6d05ec21248e07c30ed8d2aa366", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.02881, "cost_known_response_count": 12, "mean_latency_sec": 4.068802733081005, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.03006999999999999, "cost_known_response_count": 16, "mean_latency_sec": 4.11387053848739, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.028439999999999997, "cost_known_response_count": 8, "mean_latency_sec": 3.61611659375194, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 21, "accuracy": 0.875, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 3, "B": 1 }, "B": { "B": 3, "D": 1 }, "C": { "C": 3, "A": 1 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.030026666666666663, "cost_known_response_count": 24, "mean_latency_sec": 4.529721722457907, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.028660000000000005, "cost_known_response_count": 12, "mean_latency_sec": 4.190440610827257, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.028430000000000007, "cost_known_response_count": 12, "mean_latency_sec": 4.51640564983245, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 9, "accuracy": 0.75, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "C": 1 }, "B": { "B": 3, "C": 1 }, "C": { "C": 3, "B": 1 } }, "mean_api_response_cost_usd": 0.029446666666666666, "cost_known_response_count": 12, "mean_latency_sec": 5.377805622750505, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.028430000000000007, "cost_known_response_count": 12, "mean_latency_sec": 3.8955991995171644, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0287, "cost_known_response_count": 12, "mean_latency_sec": 4.26260359059476, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.028658333333333338, "cost_known_response_count": 12, "mean_latency_sec": 4.032667939929524, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.03216416666666667, "cost_known_response_count": 12, "mean_latency_sec": 4.954972806250832, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.028830000000000012, "cost_known_response_count": 24, "mean_latency_sec": 4.081929911961197, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.02895999999999999, "cost_known_response_count": 24, "mean_latency_sec": 4.620052593503108, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 9, "accuracy": 0.75, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "C": 1 }, "B": { "B": 3, "A": 1 }, "C": { "C": 3, "B": 1 } }, "mean_api_response_cost_usd": 0.029474999999999998, "cost_known_response_count": 12, "mean_latency_sec": 4.385274764858575, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.029069999999999988, "cost_known_response_count": 16, "mean_latency_sec": 4.50531098486681, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.02834999999999999, "cost_known_response_count": 12, "mean_latency_sec": 4.152867609993943, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.028769999999999993, "cost_known_response_count": 24, "mean_latency_sec": 5.020648492005421, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.028599166666666665, "cost_known_response_count": 12, "mean_latency_sec": 4.154637009671812, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.02871000000000001, "cost_known_response_count": 12, "mean_latency_sec": 4.071159572689794, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.028399999999999998, "cost_known_response_count": 12, "mean_latency_sec": 3.468038189166691, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.029088493150684944, "cost_known_response_count": 292, "mean_latency_sec": 4.353656184903827, "latency_known_response_count": 292 }, { "model_id": "claude-haiku-4-5-20251001", "model_config": { "api_key_env": "ANTHROPIC_API_KEY", "base_url": null, "display_name": "Claude Haiku 4.5", "enabled": true, "id": "claude-haiku-4-5-20251001", "input_per_m": 1.0, "max_output_tokens": 1024, "model": "claude-haiku-4-5-20251001", "notes": "Latest Haiku, using its dated snapshot ID.", "output_per_m": 5.0, "pricing_notes": "Standard online API rates in USD per million tokens; excludes caching, batch discounts and tools.", "pricing_source_url": "https://platform.claude.com/docs/en/about-claude/pricing", "provider": "anthropic", "source_url": "https://platform.claude.com/docs/en/models/overview" }, "model_config_fingerprint": "0955227c94c7dc3da8e437be80ebacb3226b826411a0b59e4d37966e425f2a3c", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.001861583333333333, "cost_known_response_count": 12, "mean_latency_sec": 0.685766983825791, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 13, "accuracy": 0.8125, "chance_accuracy": 0.25, "confusion": { "A": { "A": 3, "D": 1 }, "B": { "B": 4 }, "C": { "C": 3, "A": 1 }, "D": { "D": 3, "C": 1 } }, "mean_api_response_cost_usd": 0.0019293750000000005, "cost_known_response_count": 16, "mean_latency_sec": 0.835106244943745, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.0018329999999999998, "cost_known_response_count": 8, "mean_latency_sec": 0.7861780843668384, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 5, "accuracy": 0.20833333333333334, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 1, "B": 2, "D": 1 }, "B": { "D": 1, "A": 2, "B": 1 }, "C": { "C": 1, "A": 2, "D": 1 }, "D": { "B": 2, "D": 1, "E": 1 }, "E": { "A": 2, "E": 1, "C": 1 }, "F": { "C": 1, "B": 1, "E": 1, "A": 1 } }, "mean_api_response_cost_usd": 0.0019325833333333328, "cost_known_response_count": 24, "mean_latency_sec": 1.0051260251542165, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 5, "accuracy": 0.4166666666666667, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "C": 1, "B": 1, "A": 2 }, "B": { "A": 3, "B": 1 }, "C": { "A": 2, "C": 2 } }, "mean_api_response_cost_usd": 0.0018519999999999997, "cost_known_response_count": 12, "mean_latency_sec": 0.7058188633333581, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0018339999999999997, "cost_known_response_count": 12, "mean_latency_sec": 0.8833899294162014, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 3, "accuracy": 0.25, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "C": 1, "B": 1, "A": 2 }, "B": { "A": 3, "B": 1 }, "C": { "A": 2, "B": 2 } }, "mean_api_response_cost_usd": 0.0018551666666666664, "cost_known_response_count": 12, "mean_latency_sec": 0.8374481608431475, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0018339999999999997, "cost_known_response_count": 12, "mean_latency_sec": 1.1842800842520471, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 5, "accuracy": 0.4166666666666667, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 1, "B": 2, "C": 1 }, "B": { "A": 1, "B": 3 }, "C": { "A": 3, "C": 1 } }, "mean_api_response_cost_usd": 0.0018585833333333334, "cost_known_response_count": 12, "mean_latency_sec": 0.9213503540086094, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 3, "A": 1 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0018579999999999996, "cost_known_response_count": 12, "mean_latency_sec": 0.7908840725722257, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 4, "accuracy": 0.3333333333333333, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 1, "B": 3 }, "B": { "B": 3, "C": 1 }, "C": { "A": 2, "B": 2 } }, "mean_api_response_cost_usd": 0.0018751666666666663, "cost_known_response_count": 12, "mean_latency_sec": 0.8398179015687978, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.0018550000000000005, "cost_known_response_count": 24, "mean_latency_sec": 0.8548036311549367, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 17, "accuracy": 0.7083333333333334, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 7, "B": 1 }, "B": { "B": 5, "A": 3 }, "C": { "C": 5, "B": 2, "A": 1 } }, "mean_api_response_cost_usd": 0.0018659999999999998, "cost_known_response_count": 24, "mean_latency_sec": 0.9287218211781388, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 1, "accuracy": 0.08333333333333333, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "C": 2, "B": 2 }, "B": { "A": 4 }, "C": { "C": 1, "A": 2, "B": 1 } }, "mean_api_response_cost_usd": 0.0018831666666666667, "cost_known_response_count": 12, "mean_latency_sec": 0.8970772477817567, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 4, "accuracy": 0.25, "chance_accuracy": 0.25, "confusion": { "A": { "D": 1, "B": 1, "A": 1, "C": 1 }, "B": { "A": 1, "C": 2, "D": 1 }, "C": { "D": 1, "C": 3 }, "D": { "B": 1, "C": 2, "A": 1 } }, "mean_api_response_cost_usd": 0.001887, "cost_known_response_count": 16, "mean_latency_sec": 0.8882843320607208, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.001832, "cost_known_response_count": 12, "mean_latency_sec": 1.475296559345831, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.0018550000000000005, "cost_known_response_count": 24, "mean_latency_sec": 0.9203851480851881, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0018470000000000003, "cost_known_response_count": 12, "mean_latency_sec": 0.9674766369183393, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 7, "accuracy": 0.5833333333333334, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "B": 1 }, "B": { "B": 3, "A": 1 }, "C": { "A": 2, "C": 1, "B": 1 } }, "mean_api_response_cost_usd": 0.0018540000000000004, "cost_known_response_count": 12, "mean_latency_sec": 0.8451824634782193, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0018309999999999995, "cost_known_response_count": 12, "mean_latency_sec": 0.7972289714283155, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.0018658904109589067, "cost_known_response_count": 292, "mean_latency_sec": 0.9070300399521143, "latency_known_response_count": 292 }, { "model_id": "claude-opus-5", "model_config": { "api_key_env": "ANTHROPIC_API_KEY", "base_url": null, "display_name": "Claude Opus 5", "enabled": true, "id": "claude-opus-5", "input_per_m": 5.0, "max_output_tokens": 4096, "model": "claude-opus-5", "notes": "Latest Opus. Default model effort is high.", "output_per_m": 25.0, "pricing_notes": "Standard online API rates in USD per million tokens; excludes caching, batch discounts and tools.", "pricing_source_url": "https://platform.claude.com/docs/en/about-claude/pricing", "provider": "anthropic", "source_url": "https://platform.claude.com/docs/en/models/overview" }, "model_config_fingerprint": "6873f7552015a1feec5bf8277ba96cf05c60c4667842dddb5aa6a1dbbec3598c", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.015420000000000001, "cost_known_response_count": 12, "mean_latency_sec": 2.7538188941640933, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.0173125, "cost_known_response_count": 16, "mean_latency_sec": 3.463270061125513, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.01421, "cost_known_response_count": 8, "mean_latency_sec": 2.7142125212267274, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.019135625, "cost_known_response_count": 24, "mean_latency_sec": 4.394226851514152, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 3, "A": 1 } }, "mean_api_response_cost_usd": 0.015903333333333332, "cost_known_response_count": 12, "mean_latency_sec": 2.955328528439471, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.014204999999999997, "cost_known_response_count": 12, "mean_latency_sec": 2.3188556289921203, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 3, "A": 1 } }, "mean_api_response_cost_usd": 0.016705, "cost_known_response_count": 12, "mean_latency_sec": 3.1216509820563565, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.014204999999999997, "cost_known_response_count": 12, "mean_latency_sec": 2.3772356811678037, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.014750416666666667, "cost_known_response_count": 12, "mean_latency_sec": 2.510818065396355, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.01531083333333333, "cost_known_response_count": 12, "mean_latency_sec": 2.6655000789032783, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.016492916666666666, "cost_known_response_count": 12, "mean_latency_sec": 3.2944234496486993, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.014405, "cost_known_response_count": 24, "mean_latency_sec": 2.5365259828346702, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.015693958333333334, "cost_known_response_count": 24, "mean_latency_sec": 2.9857914192107273, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.018312916666666665, "cost_known_response_count": 12, "mean_latency_sec": 4.6849925132410135, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.0152421875, "cost_known_response_count": 16, "mean_latency_sec": 2.8817641455098055, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.014165000000000002, "cost_known_response_count": 12, "mean_latency_sec": 2.4382505043079923, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.014375000000000008, "cost_known_response_count": 24, "mean_latency_sec": 2.7489673644158756, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.015231250000000002, "cost_known_response_count": 12, "mean_latency_sec": 3.2676311345712747, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.015597083333333336, "cost_known_response_count": 12, "mean_latency_sec": 2.8071138543309644, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.014190000000000001, "cost_known_response_count": 12, "mean_latency_sec": 2.8791592866085316, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.015640582191780827, "cost_known_response_count": 292, "mean_latency_sec": 3.027752977840153, "latency_known_response_count": 292 }, { "model_id": "claude-sonnet-5", "model_config": { "api_key_env": "ANTHROPIC_API_KEY", "base_url": null, "display_name": "Claude Sonnet 5", "enabled": true, "id": "claude-sonnet-5", "input_per_m": 2.0, "max_output_tokens": 4096, "model": "claude-sonnet-5", "notes": "Latest Sonnet. Default model effort is high.", "output_per_m": 10.0, "pricing_notes": "Standard online API rates in USD per million tokens; excludes caching, batch discounts and tools.", "pricing_source_url": "https://platform.claude.com/docs/en/about-claude/pricing", "provider": "anthropic", "source_url": "https://platform.claude.com/docs/en/models/overview" }, "model_config_fingerprint": "3eea74f353075099be73c9595a094bb3f7a468c7688d17c3b5cd217c93d9bb88", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0060305, "cost_known_response_count": 12, "mean_latency_sec": 2.1173559073358774, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.006589375, "cost_known_response_count": 16, "mean_latency_sec": 2.499153832308366, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.005684000000000001, "cost_known_response_count": 8, "mean_latency_sec": 1.8981144173594657, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 23, "accuracy": 0.9583333333333334, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "F": 1, "D": 3 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.008769666666666667, "cost_known_response_count": 24, "mean_latency_sec": 4.600103180579026, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.006282166666666668, "cost_known_response_count": 12, "mean_latency_sec": 2.3185310675859605, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005682, "cost_known_response_count": 12, "mean_latency_sec": 1.9031745659206838, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 3, "B": 1 } }, "mean_api_response_cost_usd": 0.006608666666666666, "cost_known_response_count": 12, "mean_latency_sec": 2.7835786984166284, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005682, "cost_known_response_count": 12, "mean_latency_sec": 1.8859758824207045, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.006051, "cost_known_response_count": 12, "mean_latency_sec": 2.4001772511789263, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005726000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.893004090418496, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 10, "accuracy": 0.8333333333333334, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "B": 1 }, "B": { "B": 3, "C": 1 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0068688333333333336, "cost_known_response_count": 12, "mean_latency_sec": 2.732540879602311, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.0057824166666666675, "cost_known_response_count": 24, "mean_latency_sec": 1.8072307109396206, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.006629250000000001, "cost_known_response_count": 24, "mean_latency_sec": 2.434077079470929, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 3, "B": 1 } }, "mean_api_response_cost_usd": 0.007621833333333334, "cost_known_response_count": 12, "mean_latency_sec": 3.489546942165665, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 14, "accuracy": 0.875, "chance_accuracy": 0.25, "confusion": { "A": { "A": 3, "D": 1 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "A": 1, "D": 3 } }, "mean_api_response_cost_usd": 0.00591625, "cost_known_response_count": 16, "mean_latency_sec": 2.0262618268316146, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005665999999999998, "cost_known_response_count": 12, "mean_latency_sec": 2.0060516109903497, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.005750000000000002, "cost_known_response_count": 24, "mean_latency_sec": 1.6701349076271679, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005810833333333332, "cost_known_response_count": 12, "mean_latency_sec": 1.8019399973369825, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.006456333333333332, "cost_known_response_count": 12, "mean_latency_sec": 2.5071661211647247, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005676, "cost_known_response_count": 12, "mean_latency_sec": 1.7339507465755257, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.006348835616438355, "cost_known_response_count": 292, "mean_latency_sec": 2.3792623017303813, "latency_known_response_count": 292 }, { "model_id": "gemini-3.1-pro-preview", "model_config": { "api_key_env": "GEMINI_API_KEY", "base_url": null, "display_name": "Gemini 3.1 Pro Preview", "enabled": true, "id": "gemini-3.1-pro-preview", "input_per_m": 2.0, "max_output_tokens": 4096, "model": "gemini-3.1-pro-preview", "notes": "Latest documented Pro model. Preview; no announced shutdown date at verification.", "output_per_m": 12.0, "pricing_notes": "Standard online rates for prompts up to 200k tokens. Above 200k: $4 input / $18 output per million tokens. Output includes thinking tokens.", "pricing_source_url": "https://ai.google.dev/gemini-api/docs/pricing", "provider": "google", "source_url": "https://ai.google.dev/gemini-api/docs/models/gemini-3.1-pro-preview" }, "model_config_fingerprint": "6ef3a6e5b007c672a50902258a256900e6fc450affd9ab4d9373ac0ef70d0ce8", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00492, "cost_known_response_count": 12, "mean_latency_sec": 3.995124593086075, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.0054867499999999994, "cost_known_response_count": 16, "mean_latency_sec": 4.321833910958958, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.004438000000000001, "cost_known_response_count": 8, "mean_latency_sec": 3.586518619515118, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.010341, "cost_known_response_count": 24, "mean_latency_sec": 7.102203995716991, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.008121999999999999, "cost_known_response_count": 12, "mean_latency_sec": 5.597304873682636, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.003912, "cost_known_response_count": 12, "mean_latency_sec": 3.2707476584861674, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 0.6666666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 2, "B": 1, "C": 1 }, "B": { "B": 3, "C": 1 }, "C": { "C": 3, "B": 1 } }, "mean_api_response_cost_usd": 0.014761000000000002, "cost_known_response_count": 12, "mean_latency_sec": 10.216131470166147, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004294, "cost_known_response_count": 12, "mean_latency_sec": 3.645164298194383, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005139, "cost_known_response_count": 12, "mean_latency_sec": 4.286963221025265, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.006971999999999999, "cost_known_response_count": 12, "mean_latency_sec": 5.172920178010827, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": null, "cost_known_response_count": 11, "mean_latency_sec": 8.621247875494495, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.0039025, "cost_known_response_count": 24, "mean_latency_sec": 3.4317216451551453, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.006193500000000001, "cost_known_response_count": 24, "mean_latency_sec": 4.737428291045944, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 9, "accuracy": 0.75, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 3, "A": 1 }, "C": { "C": 2, "A": 1, "B": 1 } }, "mean_api_response_cost_usd": null, "cost_known_response_count": 11, "mean_latency_sec": 8.13866430457953, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 13, "accuracy": 0.8125, "chance_accuracy": 0.25, "confusion": { "A": { "A": 2, "D": 2 }, "B": { "B": 4 }, "C": { "C": 3, "D": 1 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.008830000000000001, "cost_known_response_count": 16, "mean_latency_sec": 6.44466856586223, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0047409999999999996, "cost_known_response_count": 12, "mean_latency_sec": 3.9126830641859365, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.0037750000000000006, "cost_known_response_count": 24, "mean_latency_sec": 3.205100104906402, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005935, "cost_known_response_count": 12, "mean_latency_sec": 4.263109834903541, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.006935999999999999, "cost_known_response_count": 12, "mean_latency_sec": 4.995358464909562, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005084, "cost_known_response_count": 12, "mean_latency_sec": 4.037677395923917, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": null, "cost_known_response_count": 290, "mean_latency_sec": 5.089822781715169, "latency_known_response_count": 292 }, { "model_id": "gemini-3.5-flash-lite", "model_config": { "api_key_env": "GEMINI_API_KEY", "base_url": null, "display_name": "Gemini 3.5 Flash-Lite", "enabled": true, "id": "gemini-3.5-flash-lite", "input_per_m": 0.3, "max_output_tokens": 1024, "model": "gemini-3.5-flash-lite", "notes": "Latest Flash-Lite cost-focused model.", "output_per_m": 2.5, "pricing_notes": "Standard text/image input and output rates. Output includes thinking tokens; no caching or batch discount assumed.", "pricing_source_url": "https://ai.google.dev/gemini-api/docs/pricing", "provider": "google", "source_url": "https://ai.google.dev/gemini-api/docs/models/gemini-3.5-flash-lite" }, "model_config_fingerprint": "1a87916f5fd901358d0a3564408464c0392096bd2f7b7412b727f9a9751abfe9", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0003686166666666666, "cost_known_response_count": 12, "mean_latency_sec": 1.0433681195330184, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.00038971875000000015, "cost_known_response_count": 16, "mean_latency_sec": 1.0395666734402766, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.000358875, "cost_known_response_count": 8, "mean_latency_sec": 1.135258165493724, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 14, "accuracy": 0.5833333333333334, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 2, "E": 1, "B": 1 }, "B": { "B": 3, "C": 1 }, "C": { "B": 1, "D": 1, "E": 2 }, "D": { "D": 2, "B": 1, "E": 1 }, "E": { "E": 4 }, "F": { "F": 3, "E": 1 } }, "mean_api_response_cost_usd": 0.00038775416666666666, "cost_known_response_count": 24, "mean_latency_sec": 0.8522222953324672, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 0.6666666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "A": 3, "B": 1 }, "C": { "A": 1, "C": 3 } }, "mean_api_response_cost_usd": 0.00036174999999999993, "cost_known_response_count": 12, "mean_latency_sec": 1.0079507695724412, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00035947499999999994, "cost_known_response_count": 12, "mean_latency_sec": 0.9049875802593306, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 4, "accuracy": 0.3333333333333333, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "B": 3, "A": 1 }, "B": { "B": 3, "A": 1 }, "C": { "B": 2, "A": 2 } }, "mean_api_response_cost_usd": 0.0003636750000000001, "cost_known_response_count": 12, "mean_latency_sec": 0.9094971346009212, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00035947499999999994, "cost_known_response_count": 12, "mean_latency_sec": 0.924811147589935, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 6, "accuracy": 0.5, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "C": 1, "B": 2, "A": 1 }, "B": { "C": 1, "B": 3 }, "C": { "A": 2, "C": 2 } }, "mean_api_response_cost_usd": 0.0003695, "cost_known_response_count": 12, "mean_latency_sec": 0.9986713339167181, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0003675583333333334, "cost_known_response_count": 12, "mean_latency_sec": 0.8970967633261656, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 5, "accuracy": 0.4166666666666667, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 2, "B": 2 }, "B": { "B": 3, "C": 1 }, "C": { "A": 2, "B": 2 } }, "mean_api_response_cost_usd": 0.00037087500000000006, "cost_known_response_count": 12, "mean_latency_sec": 0.9367456200901264, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.00036391250000000004, "cost_known_response_count": 24, "mean_latency_sec": 1.018560872253147, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.00036677083333333326, "cost_known_response_count": 24, "mean_latency_sec": 0.918762208413682, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 6, "accuracy": 0.5, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "C": 1 }, "B": { "B": 3, "A": 1 }, "C": { "A": 1, "B": 3 } }, "mean_api_response_cost_usd": 0.00037297499999999995, "cost_known_response_count": 12, "mean_latency_sec": 1.2075398167556461, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 4, "accuracy": 0.25, "chance_accuracy": 0.25, "confusion": { "A": { "A": 2, "C": 1, "B": 1 }, "B": { "A": 1, "C": 3 }, "C": { "A": 2, "C": 1, "D": 1 }, "D": { "D": 1, "C": 1, "A": 1, "B": 1 } }, "mean_api_response_cost_usd": 0.00037093125000000005, "cost_known_response_count": 16, "mean_latency_sec": 1.0068143043172313, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00035887500000000004, "cost_known_response_count": 12, "mean_latency_sec": 1.0741419767437037, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.0003661375000000001, "cost_known_response_count": 24, "mean_latency_sec": 0.9781237613836614, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00036559999999999983, "cost_known_response_count": 12, "mean_latency_sec": 0.9860607890974885, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0003651750000000001, "cost_known_response_count": 12, "mean_latency_sec": 0.9180815956109049, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0003617, "cost_known_response_count": 12, "mean_latency_sec": 0.9623050115963755, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.00036854178082191766, "cost_known_response_count": 292, "mean_latency_sec": 0.9777510690738842, "latency_known_response_count": 292 }, { "model_id": "gemini-3.8-flash", "model_config": { "api_key_env": "GEMINI_API_KEY", "base_url": null, "display_name": "Gemini 3.8 Flash", "enabled": true, "id": "gemini-3.8-flash", "input_per_m": 0.75, "max_output_tokens": 4096, "model": "gemini-3.8-flash", "notes": "Latest stable Flash; supports low, medium and high thinking, not minimal.", "output_per_m": 3.75, "pricing_notes": "Standard introductory rates through 2026-12-31. From 2027-01-01: $1.50 input / $7.50 output per million tokens. Output includes thinking tokens.", "pricing_source_url": "https://ai.google.dev/gemini-api/docs/pricing", "provider": "google", "source_url": "https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash" }, "model_config_fingerprint": "7de720d7bc3a5dd06ed19c22ff2574e4c880b911ce7efdfc8a824db7afb0be77", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0016220625, "cost_known_response_count": 12, "mean_latency_sec": 1.7798701879898242, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": null, "cost_known_response_count": 15, "mean_latency_sec": 1.7603205601189984, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.00116859375, "cost_known_response_count": 8, "mean_latency_sec": 1.5256814739841502, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.00228021875, "cost_known_response_count": 24, "mean_latency_sec": 2.3537753950319407, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0024415624999999997, "cost_known_response_count": 12, "mean_latency_sec": 3.1790631610977775, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0012983749999999998, "cost_known_response_count": 12, "mean_latency_sec": 1.4690236594954815, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 0.6666666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 2, "B": 1, "C": 1 }, "B": { "B": 3, "C": 1 }, "C": { "C": 3, "B": 1 } }, "mean_api_response_cost_usd": 0.0043023125, "cost_known_response_count": 12, "mean_latency_sec": 4.432985757264153, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0012958750000000002, "cost_known_response_count": 12, "mean_latency_sec": 1.601837701755964, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00145125, "cost_known_response_count": 12, "mean_latency_sec": 2.4211466713168193, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00226275, "cost_known_response_count": 12, "mean_latency_sec": 2.3545307584184534, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0031625, "cost_known_response_count": 12, "mean_latency_sec": 3.1187104968412314, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.0013060312500000001, "cost_known_response_count": 24, "mean_latency_sec": 1.7257321083258528, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.00218375, "cost_known_response_count": 24, "mean_latency_sec": 2.4521022841217928, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 9, "accuracy": 0.75, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 3, "A": 1 }, "C": { "C": 2, "A": 1, "B": 1 } }, "mean_api_response_cost_usd": 0.0039283750000000004, "cost_known_response_count": 12, "mean_latency_sec": 3.6516173172567505, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.0034098281249999997, "cost_known_response_count": 16, "mean_latency_sec": 3.1267101353078033, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0013228125, "cost_known_response_count": 12, "mean_latency_sec": 2.7733060119110937, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.0012209687499999996, "cost_known_response_count": 24, "mean_latency_sec": 1.470724560043891, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0018993125, "cost_known_response_count": 12, "mean_latency_sec": 2.102720522894136, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0019904375, "cost_known_response_count": 12, "mean_latency_sec": 2.04095675892313, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.001266125, "cost_known_response_count": 12, "mean_latency_sec": 1.7192359278269578, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": null, "cost_known_response_count": 291, "mean_latency_sec": 2.308883652243722, "latency_known_response_count": 292 }, { "model_id": "gpt-5.6-luna", "model_config": { "api_key_env": "OPENAI_API_KEY", "base_url": null, "display_name": "GPT-5.6 Luna", "enabled": true, "id": "gpt-5.6-luna", "input_per_m": 0.2, "max_output_tokens": 4096, "model": "gpt-5.6-luna", "notes": "Explicit public API model ID; uses model-default reasoning settings.", "output_per_m": 1.2, "pricing_notes": "Standard online API rates in USD per million tokens; excludes caching, batch discounts and tools.", "pricing_source_url": "https://developers.openai.com/api/docs/models", "provider": "openai", "source_url": "https://developers.openai.com/api/docs/models/gpt-5.6-luna" }, "model_config_fingerprint": "ba032b15a2527d5994d3971633f9b2f69ae89aed5cdbc739c70e747bf4acd2cd", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.000496, "cost_known_response_count": 12, "mean_latency_sec": 1.1193047641718294, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.0005697750000000002, "cost_known_response_count": 16, "mean_latency_sec": 2.1304148307535797, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.0004904000000000001, "cost_known_response_count": 8, "mean_latency_sec": 2.3022731995006325, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.0006115, "cost_known_response_count": 24, "mean_latency_sec": 2.753220321998621, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "B": 1, "A": 3 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0005025, "cost_known_response_count": 12, "mean_latency_sec": 2.0339036351651885, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0004907999999999999, "cost_known_response_count": 12, "mean_latency_sec": 1.3632298531204772, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "B": 1, "A": 3 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00055, "cost_known_response_count": 12, "mean_latency_sec": 2.5284126456729914, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0004907999999999999, "cost_known_response_count": 12, "mean_latency_sec": 1.443124400509987, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 0.6666666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 1, "B": 2, "C": 1 }, "B": { "C": 1, "B": 3 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0004954, "cost_known_response_count": 12, "mean_latency_sec": 1.7847412430952925, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0004948, "cost_known_response_count": 12, "mean_latency_sec": 1.6788918713961418, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 0.6666666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 2, "B": 2 }, "B": { "B": 3, "C": 1 }, "C": { "C": 3, "B": 1 } }, "mean_api_response_cost_usd": 0.0005282, "cost_known_response_count": 12, "mean_latency_sec": 1.7408372602658346, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.0004948, "cost_known_response_count": 24, "mean_latency_sec": 1.6008142068167217, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.0005386000000000001, "cost_known_response_count": 24, "mean_latency_sec": 2.080235448500995, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "C": 1 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0005578, "cost_known_response_count": 12, "mean_latency_sec": 2.5527162077487446, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.6875, "chance_accuracy": 0.25, "confusion": { "A": { "A": 2, "D": 2 }, "B": { "B": 2, "C": 1, "D": 1 }, "C": { "C": 4 }, "D": { "D": 3, "A": 1 } }, "mean_api_response_cost_usd": 0.0005220749999999999, "cost_known_response_count": 16, "mean_latency_sec": 1.7225169190060114, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0004900000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.8543006797456958, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.0004942000000000001, "cost_known_response_count": 24, "mean_latency_sec": 1.301580238083261, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0004927999999999999, "cost_known_response_count": 12, "mean_latency_sec": 1.6607438602562372, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0004945999999999999, "cost_known_response_count": 12, "mean_latency_sec": 1.0441570235804345, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0004902, "cost_known_response_count": 12, "mean_latency_sec": 1.2324385511747096, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.0005192397260273978, "cost_known_response_count": 292, "mean_latency_sec": 1.8156408312075056, "latency_known_response_count": 292 }, { "model_id": "gpt-5.6-sol", "model_config": { "api_key_env": "OPENAI_API_KEY", "base_url": null, "display_name": "GPT-5.6 Sol", "enabled": true, "id": "gpt-5.6-sol", "input_per_m": 4.0, "max_output_tokens": 4096, "model": "gpt-5.6-sol", "notes": "Explicit public API model ID; uses model-default reasoning settings.", "output_per_m": 20.0, "pricing_notes": "Standard online API rates in USD per million tokens; excludes caching, batch discounts and tools.", "pricing_source_url": "https://developers.openai.com/api/docs/models", "provider": "openai", "source_url": "https://developers.openai.com/api/docs/models/gpt-5.6-sol" }, "model_config_fingerprint": "e9c56323e22fab739cc8759d763c0705a149bee0b28088ae08172377899983a3", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.009864, "cost_known_response_count": 12, "mean_latency_sec": 1.4833642513452407, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.010132, "cost_known_response_count": 16, "mean_latency_sec": 1.7608325184410205, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.009751999999999999, "cost_known_response_count": 8, "mean_latency_sec": 1.7203075486031594, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.011320666666666665, "cost_known_response_count": 24, "mean_latency_sec": 2.6867915913802185, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.011223666666666667, "cost_known_response_count": 12, "mean_latency_sec": 2.640328687169434, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.009760000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.372619612928247, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.011187666666666665, "cost_known_response_count": 12, "mean_latency_sec": 3.3021124087584517, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.009760000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.5875333646545187, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.009852, "cost_known_response_count": 12, "mean_latency_sec": 1.7211906325052648, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00984, "cost_known_response_count": 12, "mean_latency_sec": 1.980754564574454, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.011027666666666667, "cost_known_response_count": 12, "mean_latency_sec": 2.8386645805148873, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.009839999999999995, "cost_known_response_count": 24, "mean_latency_sec": 1.5672840930055827, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.00988, "cost_known_response_count": 24, "mean_latency_sec": 1.729208703128582, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.01139933333333333, "cost_known_response_count": 12, "mean_latency_sec": 2.644847537002837, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 14, "accuracy": 0.875, "chance_accuracy": 0.25, "confusion": { "A": { "A": 3, "D": 1 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 3, "A": 1 } }, "mean_api_response_cost_usd": 0.01022325, "cost_known_response_count": 16, "mean_latency_sec": 1.8057278145934106, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.009744000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.665995031168374, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.009828000000000002, "cost_known_response_count": 24, "mean_latency_sec": 1.7466550991181673, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.009800000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.4491743659212564, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.009836, "cost_known_response_count": 12, "mean_latency_sec": 2.3535377709583067, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.009748000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.5636255874899991, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.010209095890410967, "cost_known_response_count": 292, "mean_latency_sec": 1.9712018977275776, "latency_known_response_count": 292 }, { "model_id": "gpt-5.6-terra", "model_config": { "api_key_env": "OPENAI_API_KEY", "base_url": null, "display_name": "GPT-5.6 Terra", "enabled": true, "id": "gpt-5.6-terra", "input_per_m": 2.0, "max_output_tokens": 4096, "model": "gpt-5.6-terra", "notes": "Explicit public API model ID; uses model-default reasoning settings.", "output_per_m": 12.0, "pricing_notes": "Standard online API rates in USD per million tokens; excludes caching, batch discounts and tools.", "pricing_source_url": "https://developers.openai.com/api/docs/models", "provider": "openai", "source_url": "https://developers.openai.com/api/docs/models/gpt-5.6-terra" }, "model_config_fingerprint": "c31dbe50cff3b0093ab5b1c1005c42ff65dbb381ff73eafe5cecac17441bcb51", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005339, "cost_known_response_count": 12, "mean_latency_sec": 1.9977472464379389, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.005478, "cost_known_response_count": 16, "mean_latency_sec": 1.8181967472410179, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.0049039999999999995, "cost_known_response_count": 8, "mean_latency_sec": 1.0844720615714323, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.005826499999999999, "cost_known_response_count": 24, "mean_latency_sec": 2.159857540129451, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005571, "cost_known_response_count": 12, "mean_latency_sec": 1.7668489951562758, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004919000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.2576169170691476, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "C": 1 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005477, "cost_known_response_count": 12, "mean_latency_sec": 1.6646474639031414, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004939000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.188663518909986, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005421999999999999, "cost_known_response_count": 12, "mean_latency_sec": 1.8417574722358647, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005214000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.3193003025856644, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005693, "cost_known_response_count": 12, "mean_latency_sec": 2.207955156239526, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.004985999999999998, "cost_known_response_count": 24, "mean_latency_sec": 1.1205010522680823, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.0055474999999999995, "cost_known_response_count": 24, "mean_latency_sec": 1.8783183209210013, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 3, "A": 1 } }, "mean_api_response_cost_usd": 0.005688000000000001, "cost_known_response_count": 12, "mean_latency_sec": 2.0422862181731034, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 10, "accuracy": 0.625, "chance_accuracy": 0.25, "confusion": { "A": { "D": 3, "A": 1 }, "B": { "C": 1, "B": 3 }, "C": { "C": 3, "D": 1 }, "D": { "D": 3, "A": 1 } }, "mean_api_response_cost_usd": 0.00599175, "cost_known_response_count": 16, "mean_latency_sec": 2.3993665873713326, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004911000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.1879804393392988, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.004942000000000001, "cost_known_response_count": 24, "mean_latency_sec": 1.216474376929303, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005214000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.4684080480559107, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005291, "cost_known_response_count": 12, "mean_latency_sec": 1.7155789891645934, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004913000000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.1060438048443757, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.005332493150684921, "cost_known_response_count": 292, "mean_latency_sec": 1.6381453276633393, "latency_known_response_count": 292 }, { "model_id": "gpt-6-astra", "model_config": { "api_key_env": "OPENAI_API_KEY", "base_url": null, "display_name": "GPT-6 Astra", "enabled": true, "id": "gpt-6-astra", "input_per_m": 10.0, "max_output_tokens": 4096, "model": "gpt-6-astra", "notes": "Explicit public API model ID; uses model-default reasoning settings.", "output_per_m": 50.0, "pricing_notes": "Standard online API rates in USD per million tokens; excludes caching, batch discounts and tools.", "pricing_source_url": "https://developers.openai.com/api/docs/models", "provider": "openai", "source_url": "https://developers.openai.com/api/docs/models/gpt-6-astra" }, "model_config_fingerprint": "70ce5d3852fae44d4189b37a379a61c6f34d59005fd8c127cad33a584c18cb3a", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.024660000000000005, "cost_known_response_count": 12, "mean_latency_sec": 2.106233872259812, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.025330000000000005, "cost_known_response_count": 16, "mean_latency_sec": 2.121859607315855, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.02438, "cost_known_response_count": 8, "mean_latency_sec": 2.1069759092497407, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.026141249999999994, "cost_known_response_count": 24, "mean_latency_sec": 2.5803771263890667, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.024579999999999994, "cost_known_response_count": 12, "mean_latency_sec": 2.1269298198167235, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0244, "cost_known_response_count": 12, "mean_latency_sec": 1.9544013783258076, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.025265, "cost_known_response_count": 12, "mean_latency_sec": 2.2157435044743274, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0244, "cost_known_response_count": 12, "mean_latency_sec": 2.221995007023603, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.024629999999999996, "cost_known_response_count": 12, "mean_latency_sec": 2.067613732506288, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.024891666666666673, "cost_known_response_count": 12, "mean_latency_sec": 1.98698683163578, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.025552500000000002, "cost_known_response_count": 12, "mean_latency_sec": 2.2855489651944176, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.0246, "cost_known_response_count": 24, "mean_latency_sec": 1.9190530999864375, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.02469999999999999, "cost_known_response_count": 24, "mean_latency_sec": 2.0276361529443725, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.025435833333333335, "cost_known_response_count": 12, "mean_latency_sec": 2.8570832524759076, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.025023750000000008, "cost_known_response_count": 16, "mean_latency_sec": 2.070454968546983, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.024359999999999996, "cost_known_response_count": 12, "mean_latency_sec": 2.1271617781118644, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.024569999999999995, "cost_known_response_count": 24, "mean_latency_sec": 2.258849301326942, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.024499999999999997, "cost_known_response_count": 12, "mean_latency_sec": 2.1864724889358818, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.02459, "cost_known_response_count": 12, "mean_latency_sec": 2.022508529238015, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.02437, "cost_known_response_count": 12, "mean_latency_sec": 1.9602046371534623, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.024865034246575317, "cost_known_response_count": 292, "mean_latency_sec": 2.1651418574970878, "latency_known_response_count": 292 }, { "model_id": "muse-spark-1.2", "model_config": { "api_key_env": "MODEL_API_KEY", "base_url": "https://api.meta.ai/v1", "display_name": "Muse Spark 1.2", "enabled": true, "id": "muse-spark-1.2", "input_per_m": 1.25, "max_output_tokens": 16384, "model": "muse-spark-1.2", "notes": "Meta Muse Spark 1.2 Standard tier via the OpenAI-compatible Meta Model API Responses endpoint. Reasoning model; uses model-default reasoning settings. 16384 output cap matches 1.3 so both Muse configurations share identical inference settings. Same $1.25/$4.25 standard pricing as 1.3; prompts never train Meta models.", "output_per_m": 4.25, "pricing_notes": "Standard tier rates in USD per million tokens; cached input $0.15/M not assumed. Output includes reasoning tokens.", "pricing_source_url": "https://dev.meta.ai/docs/pricing-rate-limits", "provider": "openai", "source_url": "https://dev.meta.ai/docs/models" }, "model_config_fingerprint": "24aa8541bfef7a62fbb06d7bb882a0cb7acc8341dc8f956b52124454dddf4967", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0046738125, "cost_known_response_count": 12, "mean_latency_sec": 2.506259565600582, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.0056314687499999985, "cost_known_response_count": 16, "mean_latency_sec": 3.6207749257591786, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.0038534375, "cost_known_response_count": 8, "mean_latency_sec": 1.5366123637359124, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.010928937500000001, "cost_known_response_count": 24, "mean_latency_sec": 9.91871589854903, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00832075, "cost_known_response_count": 12, "mean_latency_sec": 7.35066720007065, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0038856875000000007, "cost_known_response_count": 12, "mean_latency_sec": 1.3875110327498987, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 9, "accuracy": 0.75, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 2, "B": 1, "C": 1 }, "B": { "B": 3, "C": 1 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.013199395833333334, "cost_known_response_count": 12, "mean_latency_sec": 13.522226308259027, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004052145833333333, "cost_known_response_count": 12, "mean_latency_sec": 1.6168189671006985, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005228229166666667, "cost_known_response_count": 12, "mean_latency_sec": 3.255382641267109, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.006302020833333334, "cost_known_response_count": 12, "mean_latency_sec": 4.64052852083114, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 3, "C": 1 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.011202770833333334, "cost_known_response_count": 12, "mean_latency_sec": 10.884028296917677, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.004522864583333334, "cost_known_response_count": 24, "mean_latency_sec": 2.7851713719865074, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.006354364583333333, "cost_known_response_count": 24, "mean_latency_sec": 4.409445599861404, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 0.6666666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "B": 1 }, "B": { "B": 3, "C": 1 }, "C": { "C": 2, "A": 1, "B": 1 } }, "mean_api_response_cost_usd": 0.012425645833333332, "cost_known_response_count": 12, "mean_latency_sec": 12.488607266151424, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 13, "accuracy": 0.8125, "chance_accuracy": 0.25, "confusion": { "A": { "A": 3, "D": 1 }, "B": { "B": 3, "A": 1 }, "C": { "C": 4 }, "D": { "D": 3, "A": 1 } }, "mean_api_response_cost_usd": 0.012767890625, "cost_known_response_count": 16, "mean_latency_sec": 11.48553496907698, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004540854166666666, "cost_known_response_count": 12, "mean_latency_sec": 2.4812600086637153, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.00413378125, "cost_known_response_count": 24, "mean_latency_sec": 1.710513457544342, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005021083333333334, "cost_known_response_count": 12, "mean_latency_sec": 3.3365402196650393, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005378520833333333, "cost_known_response_count": 12, "mean_latency_sec": 3.307549260188049, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004007666666666667, "cost_known_response_count": 12, "mean_latency_sec": 2.0714202810292286, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.006872051369863007, "cost_known_response_count": 292, "mean_latency_sec": 5.246410150410231, "latency_known_response_count": 292 }, { "model_id": "muse-spark-1.3", "model_config": { "api_key_env": "MODEL_API_KEY", "base_url": "https://api.meta.ai/v1", "display_name": "Muse Spark 1.3", "enabled": true, "id": "muse-spark-1.3", "input_per_m": 1.25, "max_output_tokens": 16384, "model": "muse-spark-1.3", "notes": "Meta Muse Spark 1.3 Standard tier via the OpenAI-compatible Meta Model API Responses endpoint. Reasoning model at model-default high effort; uses model-default reasoning settings. 16384 output cap: the 4096-cap probe returned status incomplete with all 4096 tokens spent on reasoning, and live outputs reach ~6900 tokens, so 16384 keeps headroom above observed reasoning volume. Same $1.25/$4.25 standard pricing as 1.2; prompts never train Meta models.", "output_per_m": 4.25, "pricing_notes": "Standard tier rates in USD per million tokens; cached input $0.15/M not assumed. Output includes reasoning tokens.", "pricing_source_url": "https://dev.meta.ai/docs/pricing-rate-limits", "provider": "openai", "source_url": "https://dev.meta.ai/docs/models" }, "model_config_fingerprint": "d501fb0c8f92e13a48222500e057dda2fad8750852df8848583d413832d05bb5", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004751729166666666, "cost_known_response_count": 12, "mean_latency_sec": 4.469805223178507, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.0059316249999999985, "cost_known_response_count": 16, "mean_latency_sec": 6.539424937684089, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.0041036562500000005, "cost_known_response_count": 8, "mean_latency_sec": 2.5624710710981162, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.009149604166666665, "cost_known_response_count": 24, "mean_latency_sec": 14.492107479163678, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.007613125000000001, "cost_known_response_count": 12, "mean_latency_sec": 12.160626556908634, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0043790416666666665, "cost_known_response_count": 12, "mean_latency_sec": 2.8594084445697567, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 10, "accuracy": 0.8333333333333334, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "C": 1 }, "B": { "B": 3, "C": 1 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.008930979166666665, "cost_known_response_count": 12, "mean_latency_sec": 14.927775024271492, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004303958333333333, "cost_known_response_count": 12, "mean_latency_sec": 2.69808476997423, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.005027770833333334, "cost_known_response_count": 12, "mean_latency_sec": 4.802917063915326, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0062694375, "cost_known_response_count": 12, "mean_latency_sec": 8.516341213486157, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.007656854166666668, "cost_known_response_count": 12, "mean_latency_sec": 11.018206482423315, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.00452853125, "cost_known_response_count": 24, "mean_latency_sec": 3.7008291647507576, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.006630260416666668, "cost_known_response_count": 24, "mean_latency_sec": 9.309789769594014, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 9, "accuracy": 0.75, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "C": 1 }, "B": { "B": 4 }, "C": { "C": 2, "A": 2 } }, "mean_api_response_cost_usd": 0.009281708333333335, "cost_known_response_count": 12, "mean_latency_sec": 18.224358616183356, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 0.75, "chance_accuracy": 0.25, "confusion": { "A": { "A": 3, "D": 1 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "C": 1, "D": 1, "A": 2 } }, "mean_api_response_cost_usd": 0.013874484375000001, "cost_known_response_count": 16, "mean_latency_sec": 25.402768941952672, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004731395833333333, "cost_known_response_count": 12, "mean_latency_sec": 3.596724639928046, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.004220375000000001, "cost_known_response_count": 24, "mean_latency_sec": 2.9515762822848046, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.004634687499999999, "cost_known_response_count": 12, "mean_latency_sec": 3.970508391134596, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0055329375000000005, "cost_known_response_count": 12, "mean_latency_sec": 6.021279079946301, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0044680833333333335, "cost_known_response_count": 12, "mean_latency_sec": 3.434802746264419, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.006402047945205482, "cost_known_response_count": 292, "mean_latency_sec": 8.297562309480208, "latency_known_response_count": 292 } ], "attempts": [ { "sequence": 1, "attempt_id": "2a0fa5d8-883e-465f-9de9-c7abcc414882", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.8139414549805224, "model_name": "gemini-3.8-flash", "output_tokens": 116, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:42.786309+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00129, "created_at": "2026-09-17T03:08:42.786Z" }, { "sequence": 2, "attempt_id": "b88d8ea8-f61d-459c-8062-20024fe2eff1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-003", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 2.162442251923494, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:08:43.118298+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T03:08:43.118Z" }, { "sequence": 3, "attempt_id": "c44e77f2-34f4-473a-820b-ada40c6823af", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.4890668659936637, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:43.450282+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:08:43.450Z" }, { "sequence": 4, "attempt_id": "bf627e10-2268-401c-806a-0f12d29e3dde", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.4879448249703273, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:43.465779+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:08:43.466Z" }, { "sequence": 5, "attempt_id": "c735caf6-c798-4705-93e4-f30427e22355", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-003", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.9630382580216974, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:08:43.769746+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:08:43.769Z" }, { "sequence": 6, "attempt_id": "931cf35d-f0d6-4856-ba46-4cfd353ae669", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006126, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.891954972059466, "model_name": "claude-sonnet-5", "output_tokens": 50, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:43.844545+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006126, "created_at": "2026-09-17T03:08:43.844Z" }, { "sequence": 7, "attempt_id": "04f622db-604f-47c1-a7b6-67eaccdda9b7", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 3.197108093998395, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:44.145387+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01434, "created_at": "2026-09-17T03:08:44.145Z" }, { "sequence": 8, "attempt_id": "822f0fbd-3d40-4171-8680-2bbc36be2347", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 3.419038404012099, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:44.377399+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:08:44.377Z" }, { "sequence": 9, "attempt_id": "2aa548ec-e126-4111-a54d-41d23fc2c95f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00547, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 3.7052989000221714, "model_name": "gpt-5.6-terra", "output_tokens": 57, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:44.668951+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00547, "created_at": "2026-09-17T03:08:44.669Z" }, { "sequence": 10, "attempt_id": "b066fac8-4087-4e54-a17e-661ba2715df3", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.9188094690907747, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:08:45.078487+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001884, "created_at": "2026-09-17T03:08:45.078Z" }, { "sequence": 11, "attempt_id": "99f77b4a-237b-433c-8024-d74d87160b07", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 4.312621732009575, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:45.258527+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T03:08:45.258Z" }, { "sequence": 12, "attempt_id": "7ade78ce-b052-4224-b7cc-ef99c81b9be8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005556, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.658496511052363, "model_name": "gemini-3.1-pro-preview", "output_tokens": 273, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:08:45.627782+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005556, "created_at": "2026-09-17T03:08:45.627Z" }, { "sequence": 13, "attempt_id": "1b699219-f4b7-42b0-bf8e-d3394c52f98c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02925, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 2.8726766069885343, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:46.355449+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02925, "created_at": "2026-09-17T03:08:46.355Z" }, { "sequence": 14, "attempt_id": "5e6b3ef3-300a-4dd1-a7ca-e69a0454bd12", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.1177445750217885, "model_name": "gpt-5.6-sol", "output_tokens": 70, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:46.801022+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011056, "created_at": "2026-09-17T03:08:46.801Z" }, { "sequence": 15, "attempt_id": "a28b0746-638b-4042-944c-92a1b7d7ad21", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005644, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.9501131169963628, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:47.043665+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005644, "created_at": "2026-09-17T03:08:47.043Z" }, { "sequence": 16, "attempt_id": "1ee8b3b9-7c99-4d5c-9553-16f561dd7985", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02609, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.743736026925035, "model_name": "gpt-6-astra", "output_tokens": 39, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:47.135667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02609, "created_at": "2026-09-17T03:08:47.135Z" }, { "sequence": 17, "attempt_id": "4d91b40a-e0dd-41c0-8054-79778e00ab08", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000568, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.9567592949606478, "model_name": "gpt-5.6-luna", "output_tokens": 71, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:47.230412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000568, "created_at": "2026-09-17T03:08:47.230Z" }, { "sequence": 18, "attempt_id": "be9ccf73-7121-4514-8a68-9c036332e76c", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.7955767150269821, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:08:47.611083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003761, "created_at": "2026-09-17T03:08:47.611Z" }, { "sequence": 19, "attempt_id": "c9998155-0186-4c5d-9f91-5209bd2a6bb6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006318, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.2516940699424595, "model_name": "muse-spark-1.2", "output_tokens": 741, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:47.717851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006318, "created_at": "2026-09-17T03:08:47.718Z" }, { "sequence": 20, "attempt_id": "e0c11b6f-2f05-4a42-9796-b1f9c9b3dbe6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004992, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 5.405047936015762, "model_name": "muse-spark-1.3", "output_tokens": 429, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:48.544154+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004992, "created_at": "2026-09-17T03:08:48.544Z" }, { "sequence": 21, "attempt_id": "af849334-8ba3-49f3-b145-1a28b0443f2b", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00247275, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 2.245866143028252, "model_name": "gemini-3.8-flash", "output_tokens": 427, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:08:48.616663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00247275, "created_at": "2026-09-17T03:08:48.616Z" }, { "sequence": 22, "attempt_id": "e7ee93f8-c5c5-4405-a134-41daaa774424", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009456, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 5.19592324597761, "model_name": "claude-sonnet-5", "output_tokens": 372, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:49.054107+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009456, "created_at": "2026-09-17T03:08:49.054Z" }, { "sequence": 23, "attempt_id": "0ad0c73f-aa4d-483d-b18f-90bd517a580f", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9859111880650744, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:49.611712+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:08:49.611Z" }, { "sequence": 24, "attempt_id": "91c53668-bafb-4348-8dc2-a048d541c40c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.1847796459915116, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:49.917175+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:08:49.917Z" }, { "sequence": 25, "attempt_id": "f24188a4-ecb1-4038-ae2d-b893b05a5139", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.4572578649967909, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:50.533957+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:08:50.534Z" }, { "sequence": 26, "attempt_id": "43dc4b09-9a92-4304-a278-ab972ea1fbd6", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.2908012910047546, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:50.551585+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:08:50.551Z" }, { "sequence": 27, "attempt_id": "ddc17511-a276-4d50-9620-e3521e715b4b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.8146571610122919, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:50.750257+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:08:50.750Z" }, { "sequence": 28, "attempt_id": "40e46d25-d884-40e3-bef6-7462c6732e1c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2261585230007768, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:50.853028+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:08:50.853Z" }, { "sequence": 29, "attempt_id": "f8eb4bb0-e5d8-4ba0-8040-9b0c434b6aa1", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": true, "cost_usd": 0.06658800000000001, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 5.940043163951486, "model_name": "gemini-3.1-pro-preview", "output_tokens": 426, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:51.583136+00:00", "request_attempts": 2, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 1, "valid": true, "within_bands": null }, "cost_usd": 0.06658800000000001, "created_at": "2026-09-17T03:08:51.583Z" }, { "sequence": 30, "attempt_id": "352ded98-d04a-42c0-bb89-ae9ca2a09802", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.0073577360017225, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:08:51.772730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:08:51.772Z" }, { "sequence": 31, "attempt_id": "a9ac0ad1-a767-4d7a-a411-4195fb452592", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00127275, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.6010703149950132, "model_name": "gemini-3.8-flash", "output_tokens": 116, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:52.167297+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00127275, "created_at": "2026-09-17T03:08:52.167Z" }, { "sequence": 32, "attempt_id": "f5d0b0fd-3a95-4b54-b5ab-b2894d59cc73", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.5507899520453066, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:52.182679+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:08:52.182Z" }, { "sequence": 33, "attempt_id": "5b9c7a35-7530-4baa-a1d2-7cce86fdebd0", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 3.843994616996497, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:52.402626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:08:52.402Z" }, { "sequence": 34, "attempt_id": "0a7e2d15-9210-4bd6-8edb-f3b5e3c5d5c7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003857, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.384220514097251, "model_name": "muse-spark-1.2", "output_tokens": 169, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:52.990691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003857, "created_at": "2026-09-17T03:08:52.990Z" }, { "sequence": 35, "attempt_id": "48cbb426-4cae-4db6-9ab0-3a342c8d2c7c", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.7949462640099227, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:53.213604+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:08:53.213Z" }, { "sequence": 36, "attempt_id": "e0793ecc-6329-4f43-adc1-db5007c09a96", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00839275, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 6.587025379994884, "model_name": "muse-spark-1.2", "output_tokens": 1223, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:53.737092+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00839275, "created_at": "2026-09-17T03:08:53.737Z" }, { "sequence": 37, "attempt_id": "a863b8e2-b057-4a64-864b-04c2bfcdaac0", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8097283049719408, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:54.008403+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:08:54.008Z" }, { "sequence": 38, "attempt_id": "4c3123a3-2617-461a-a5a3-e8e32521626f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00631025, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 7.220046381931752, "model_name": "muse-spark-1.3", "output_tokens": 733, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:54.278009+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00631025, "created_at": "2026-09-17T03:08:54.278Z" }, { "sequence": 39, "attempt_id": "8e6f3e5e-af26-4b76-9f94-2ce50ef6333e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0807715359842405, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:54.309923+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:08:54.310Z" }, { "sequence": 40, "attempt_id": "aac3f068-2d4a-48c9-946d-8d2ee01ed3ef", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.1587923150509596, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:54.341737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:08:54.341Z" }, { "sequence": 41, "attempt_id": "01b4e958-818c-4451-9678-90c65663a39b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00462625, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.7478385599097237, "model_name": "muse-spark-1.3", "output_tokens": 350, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:54.616274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00462625, "created_at": "2026-09-17T03:08:54.616Z" }, { "sequence": 42, "attempt_id": "d57e7f97-f6a1-4231-94a7-5b1abdf6aaf6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0044728110078722, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:55.028738+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:08:55.028Z" }, { "sequence": 43, "attempt_id": "e67e5194-bf04-4e58-a0f6-df06fd7bd61a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.0822818420128897, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:55.088761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:08:55.088Z" }, { "sequence": 44, "attempt_id": "2d121f77-972a-48a1-ac8a-177405eb7d5d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006494, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 4.731778824003413, "model_name": "gemini-3.1-pro-preview", "output_tokens": 355, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:55.283701+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006494, "created_at": "2026-09-17T03:08:55.283Z" }, { "sequence": 45, "attempt_id": "782f5d63-8b80-47b6-8626-6a3d36aada66", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.6146749620093033, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:55.367092+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:08:55.367Z" }, { "sequence": 46, "attempt_id": "5ed9158c-0940-42a2-87ea-e931fc562797", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.323662888025865, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:08:55.680068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:08:55.680Z" }, { "sequence": 47, "attempt_id": "94f5ff5c-b5e2-44b1-8a95-7d17af6a6a8c", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122775, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.4182926309294999, "model_name": "gemini-3.8-flash", "output_tokens": 104, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:08:55.742912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00122775, "created_at": "2026-09-17T03:08:55.743Z" }, { "sequence": 48, "attempt_id": "a80381df-d4c8-447e-97db-d25bc9360fcb", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.055100120021962, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:55.843441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:08:55.843Z" }, { "sequence": 49, "attempt_id": "819b3212-9c8c-4e93-9f02-3e0da882e081", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00393775, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.2942044719820842, "model_name": "muse-spark-1.2", "output_tokens": 188, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:56.338644+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00393775, "created_at": "2026-09-17T03:08:56.338Z" }, { "sequence": 50, "attempt_id": "acad22fa-8201-4898-bb0d-27ea63d015a5", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.9657453109975904, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:56.661668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T03:08:56.661Z" }, { "sequence": 51, "attempt_id": "a2fd3506-4917-4587-8d93-d03d01d55fa2", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003218, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.8672300160396844, "model_name": "gemini-3.1-pro-preview", "output_tokens": 82, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:57.160747+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003218, "created_at": "2026-09-17T03:08:57.160Z" }, { "sequence": 52, "attempt_id": "b63aa8ab-ca13-430c-958b-31fab2b36658", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.0020068070152774, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.356827+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004946, "created_at": "2026-09-17T03:08:57.357Z" }, { "sequence": 53, "attempt_id": "aa09f6bc-83b3-407a-9ab9-6c8618f2daa9", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01844, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 13.660711804986931, "model_name": "claude-opus-5", "output_tokens": 164, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.444609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01844, "created_at": "2026-09-17T03:08:57.444Z" }, { "sequence": 54, "attempt_id": "c9cf2fe2-d862-4aee-9664-138fda2b581d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.8308793450705707, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.589351+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:08:57.589Z" }, { "sequence": 55, "attempt_id": "0a44fd7b-043f-424a-8500-b6700838faf9", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006218, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.1986675900407135, "model_name": "claude-sonnet-5", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.608931+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006218, "created_at": "2026-09-17T03:08:57.609Z" }, { "sequence": 56, "attempt_id": "4938798e-1678-4aa1-813a-e52849c3a7c3", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014345, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.296391654992476, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.623667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014345, "created_at": "2026-09-17T03:08:57.623Z" }, { "sequence": 57, "attempt_id": "cf270fbc-22f1-46cf-bdbe-090d2d445abe", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.196143144974485, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.874375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:08:57.874Z" }, { "sequence": 58, "attempt_id": "ed0ba4d6-19cc-41ca-a420-babf59e6c656", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004503, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.327036952949129, "model_name": "muse-spark-1.3", "output_tokens": 321, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:57.958744+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004503, "created_at": "2026-09-17T03:08:57.958Z" }, { "sequence": 59, "attempt_id": "d8ced39e-b7a9-491c-b97a-7ea513672d9a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.4070883790263906, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:58.266577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:08:58.266Z" }, { "sequence": 60, "attempt_id": "1ce6b080-c625-4e80-a2ac-a0b9e709e7a3", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.8184933529701084, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:08:58.283671+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003683, "created_at": "2026-09-17T03:08:58.283Z" }, { "sequence": 61, "attempt_id": "00dd3bc2-b5bf-43b5-80b9-ac492400aa10", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.580396836041473, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:58.684812+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T03:08:58.684Z" }, { "sequence": 62, "attempt_id": "99551957-47fb-4ece-96cf-5c7c74e5cff8", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001662, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.1478824509540573, "model_name": "gemini-3.8-flash", "output_tokens": 216, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:59.521195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001662, "created_at": "2026-09-17T03:08:59.521Z" }, { "sequence": 63, "attempt_id": "d829b07b-6c66-4e5a-b1ef-b6ab66f21a7f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001925, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 1.4491039759013802, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:59.733120+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001925, "created_at": "2026-09-17T03:08:59.733Z" }, { "sequence": 64, "attempt_id": "3086990d-3d1b-464c-9f33-78cbd28aedfb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.114392967079766, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:59.815716+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:08:59.815Z" }, { "sequence": 65, "attempt_id": "beba0026-d431-4ed8-9b61-c1d762803f69", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.0260190500412136, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:59.920273+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015025, "created_at": "2026-09-17T03:08:59.920Z" }, { "sequence": 66, "attempt_id": "d67d99c2-e7ff-4767-baf7-35377080d05a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00649, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.9516219070646912, "model_name": "claude-sonnet-5", "output_tokens": 59, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:59.938082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00649, "created_at": "2026-09-17T03:08:59.938Z" }, { "sequence": 67, "attempt_id": "099ae002-ff80-407a-a408-7b514cccc852", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.7864581609610468, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:00.085607+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:09:00.085Z" }, { "sequence": 68, "attempt_id": "6b4e6053-e7e3-4eda-8136-e51d8a7facc3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00535975, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 2.989831362036057, "model_name": "muse-spark-1.2", "output_tokens": 517, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:00.628514+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00535975, "created_at": "2026-09-17T03:09:00.628Z" }, { "sequence": 69, "attempt_id": "51078d91-a5a4-4d4c-bf69-b20023f4692b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.209222075995058, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:00.850620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:09:00.850Z" }, { "sequence": 70, "attempt_id": "d05b654d-a3e8-4b7c-a0ea-ab5a16425a6a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005346, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.3640940129989758, "model_name": "gpt-5.6-terra", "output_tokens": 35, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:00.902460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005346, "created_at": "2026-09-17T03:09:00.902Z" }, { "sequence": 71, "attempt_id": "7493aeb9-ff55-4be5-8444-b35b42ce9b0e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005526, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1972261030459777, "model_name": "gpt-5.6-luna", "output_tokens": 50, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:00.946998+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005526, "created_at": "2026-09-17T03:09:00.947Z" }, { "sequence": 72, "attempt_id": "2074d84f-2067-4857-bb42-df99a62e27ec", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.0346066299825907, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:00.991639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:09:00.991Z" }, { "sequence": 73, "attempt_id": "89b3ade5-8d82-4cb1-8b79-7b0e22c1a24a", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00182625, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.5604766469914466, "model_name": "gemini-3.8-flash", "output_tokens": 245, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:01.498900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00182625, "created_at": "2026-09-17T03:09:01.499Z" }, { "sequence": 74, "attempt_id": "60e39d66-8d22-4560-8aee-50e5ea5d48f4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006088, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.422139151021838, "model_name": "gemini-3.1-pro-preview", "output_tokens": 318, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:01.601218+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006088, "created_at": "2026-09-17T03:09:01.601Z" }, { "sequence": 75, "attempt_id": "639830f3-8319-4674-bbb4-2eb857c3feab", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6923633040860295, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:01.700512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:09:01.700Z" }, { "sequence": 76, "attempt_id": "c1ef958e-cb04-4440-8fff-b0df6b4cc196", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004184, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.0085124240722507, "model_name": "gemini-3.1-pro-preview", "output_tokens": 147, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:02.840598+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004184, "created_at": "2026-09-17T03:09:02.840Z" }, { "sequence": 77, "attempt_id": "e27b8c69-578c-4f2a-a0fb-f9eab870db34", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00504125, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.4593563420930877, "model_name": "muse-spark-1.2", "output_tokens": 420, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:03.107716+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00504125, "created_at": "2026-09-17T03:09:03.107Z" }, { "sequence": 78, "attempt_id": "5136a5bc-14ea-4f63-9f2e-ae2bd19a1f12", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006218, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.1516330160666257, "model_name": "claude-sonnet-5", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:03.126041+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006218, "created_at": "2026-09-17T03:09:03.126Z" }, { "sequence": 79, "attempt_id": "9c01b575-9e72-4494-89a4-7e6d85e45c4d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0056445, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 5.909804499009624, "model_name": "muse-spark-1.3", "output_tokens": 584, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:03.519059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0056445, "created_at": "2026-09-17T03:09:03.519Z" }, { "sequence": 80, "attempt_id": "a47d92ef-efc4-4358-adba-ce49d7da441c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.0162298530340195, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:03.537263+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:09:03.538Z" }, { "sequence": 81, "attempt_id": "0e4fb0dc-0646-4625-b6ba-46e2e636066a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9405328630236909, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:03.558076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:09:03.558Z" }, { "sequence": 82, "attempt_id": "90bc9051-bb0e-43b4-aecd-c6dafc98c2ed", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00528, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9320375339593738, "model_name": "gpt-5.6-terra", "output_tokens": 40, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:03.650337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00528, "created_at": "2026-09-17T03:09:03.650Z" }, { "sequence": 83, "attempt_id": "520b1b90-2087-44b4-b7a1-f247051eda9c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.2624363750219345, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:04.130334+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:09:04.130Z" }, { "sequence": 84, "attempt_id": "936dc9ce-29e5-48f3-b436-9d83832b5ba6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000552, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.3357815931085497, "model_name": "gpt-5.6-luna", "output_tokens": 60, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:04.194419+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000552, "created_at": "2026-09-17T03:09:04.194Z" }, { "sequence": 85, "attempt_id": "afd5d749-0b86-46a9-8f69-1c2b567a7f1e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.8754926429828629, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:09:04.413082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:09:04.413Z" }, { "sequence": 86, "attempt_id": "4da01257-5ed0-4521-ab65-00137f174ecf", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.9114576099673286, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:04.830759+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01447, "created_at": "2026-09-17T03:09:04.830Z" }, { "sequence": 87, "attempt_id": "8baf65f9-cdd9-4f5c-b70e-fef70f426d07", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0016125, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.8058915979927406, "model_name": "gemini-3.8-flash", "output_tokens": 200, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:09:04.949659+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0016125, "created_at": "2026-09-17T03:09:04.949Z" }, { "sequence": 88, "attempt_id": "423e36c1-3519-4221-9d24-6330a9cf8462", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7052383120171726, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:05.140568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:09:05.140Z" }, { "sequence": 89, "attempt_id": "b801c80d-3a79-44d6-838b-b9fadad3b843", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004991, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 2.4157997770234942, "model_name": "muse-spark-1.2", "output_tokens": 427, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:05.989735+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004991, "created_at": "2026-09-17T03:09:05.989Z" }, { "sequence": 90, "attempt_id": "92645036-36d5-4cf7-bab8-5f0ac6db827c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005804, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.865882856072858, "model_name": "gemini-3.1-pro-preview", "output_tokens": 292, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:06.992248+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005804, "created_at": "2026-09-17T03:09:06.992Z" }, { "sequence": 91, "attempt_id": "4d1799f3-5221-4314-93ee-f5e483d2c892", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006192, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.3037946169497445, "model_name": "gpt-5.6-terra", "output_tokens": 105, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:07.509274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006192, "created_at": "2026-09-17T03:09:07.509Z" }, { "sequence": 92, "attempt_id": "fa4f582f-2617-4862-9a13-0c5ee73823f3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00574675, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 7.440627490985207, "model_name": "muse-spark-1.3", "output_tokens": 586, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:07.543740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00574675, "created_at": "2026-09-17T03:09:07.543Z" }, { "sequence": 93, "attempt_id": "e09621af-81e9-4c7c-96ea-93866fef7a97", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02706, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.7393484000349417, "model_name": "gpt-6-astra", "output_tokens": 48, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:07.659815+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02706, "created_at": "2026-09-17T03:09:07.659Z" }, { "sequence": 94, "attempt_id": "caa04fbd-e86f-424b-be64-5210777ced0c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011284, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.7015887949382886, "model_name": "gpt-5.6-sol", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:07.731886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011284, "created_at": "2026-09-17T03:09:07.732Z" }, { "sequence": 95, "attempt_id": "b0a30887-f8c3-40ee-b953-2a85874d8614", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017845, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.7716471750754863, "model_name": "claude-opus-5", "output_tokens": 127, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:07.921100+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017845, "created_at": "2026-09-17T03:09:07.921Z" }, { "sequence": 96, "attempt_id": "7cbfc6ee-fdf4-44e6-9f46-a8517057c581", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-022", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8845387200126424, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:09:08.447310+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:09:08.447Z" }, { "sequence": 97, "attempt_id": "c4012699-df49-4b6c-a3d4-136188a6c358", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006216, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.6831432330654934, "model_name": "gpt-5.6-luna", "output_tokens": 107, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:08.691793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006216, "created_at": "2026-09-17T03:09:08.691Z" }, { "sequence": 98, "attempt_id": "ff31a3b3-e853-43ff-965f-9ef19d877fd5", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 6.844648545025848, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:10.512740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:09:10.512Z" }, { "sequence": 99, "attempt_id": "9954d8da-2382-402c-977d-6e14df3b70bc", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.8498593580443412, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:10.561155+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005758, "created_at": "2026-09-17T03:09:10.561Z" }, { "sequence": 100, "attempt_id": "b6a39b4c-f5e7-4ef1-a534-80dbb28ce31a", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00330225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 3.0507228209171444, "model_name": "gemini-3.8-flash", "output_tokens": 638, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:10.582619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00330225, "created_at": "2026-09-17T03:09:10.582Z" }, { "sequence": 101, "attempt_id": "d7ac7491-1e7a-4efb-beda-5ed691b54105", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.6815145130967721, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:11.215395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T03:09:11.215Z" }, { "sequence": 102, "attempt_id": "6b538b77-9386-45f2-9d21-2e9088f9a6cc", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.7935756780207157, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:11.260023+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014395, "created_at": "2026-09-17T03:09:11.260Z" }, { "sequence": 103, "attempt_id": "d56e281c-7fb8-408f-94ef-648895cecafd", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.772584279999137, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:11.712381+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T03:09:11.712Z" }, { "sequence": 104, "attempt_id": "95ffa6cb-e497-4c29-8d1a-9b25d3312381", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.1984361249487847, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:11.800466+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:09:11.800Z" }, { "sequence": 105, "attempt_id": "ba964682-eb47-4ce6-bf4f-348f7c23772f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011578, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 7.602495500002988, "model_name": "claude-sonnet-5", "output_tokens": 571, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:11.823168+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011578, "created_at": "2026-09-17T03:09:11.823Z" }, { "sequence": 106, "attempt_id": "d2e4aac0-bede-4881-87b6-ba0f02256038", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.7487130539957434, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:12.036528+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:09:12.036Z" }, { "sequence": 107, "attempt_id": "1b13f5d9-23b8-4026-9cbe-30c17f5d9989", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.7596855679294094, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:12.342678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:09:12.342Z" }, { "sequence": 108, "attempt_id": "dfbbc386-447d-4b4d-b825-adfa28f860c2", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003582, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.1037840569624677, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:12.947133+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003582, "created_at": "2026-09-17T03:09:12.947Z" }, { "sequence": 109, "attempt_id": "4d3eca3b-97c4-4ead-b5ca-5db1aecd8eee", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00150675, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.7214524009032175, "model_name": "gemini-3.8-flash", "output_tokens": 173, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:13.544943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00150675, "created_at": "2026-09-17T03:09:13.545Z" }, { "sequence": 110, "attempt_id": "fedcab1f-608b-4a4f-9aa6-cfc0c8f8a92d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010442, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.798644572030753, "model_name": "gemini-3.1-pro-preview", "output_tokens": 668, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:13.809279+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010442, "created_at": "2026-09-17T03:09:13.809Z" }, { "sequence": 111, "attempt_id": "4b2de895-31d7-4d15-96bf-dd5a59801eed", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00458775, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.7344858230790123, "model_name": "muse-spark-1.2", "output_tokens": 333, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:15.108012+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00458775, "created_at": "2026-09-17T03:09:15.108Z" }, { "sequence": 112, "attempt_id": "afc48e2a-016f-4156-8f8b-9bdfe3678a0a", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005108, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 3.9145725070266053, "model_name": "gemini-3.1-pro-preview", "output_tokens": 235, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:15.656844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005108, "created_at": "2026-09-17T03:09:15.657Z" }, { "sequence": 113, "attempt_id": "edbb2a9f-2ef0-455d-8050-81afe18babfe", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0043285, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.8232133450219408, "model_name": "muse-spark-1.3", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:15.889684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0043285, "created_at": "2026-09-17T03:09:15.889Z" }, { "sequence": 114, "attempt_id": "02f4ba2b-7fad-4a14-87c8-44b0650c45be", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01557, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.383471436914988, "model_name": "claude-opus-5", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:15.991775+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01557, "created_at": "2026-09-17T03:09:15.992Z" }, { "sequence": 115, "attempt_id": "11d862c6-5dfb-4c15-820b-95cc03a79dd3", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 2.9290942930383608, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:16.010287+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02901, "created_at": "2026-09-17T03:09:16.010Z" }, { "sequence": 116, "attempt_id": "897d285e-3d28-4b78-a81e-56b144c1910b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005296, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 4.800491377944127, "model_name": "gpt-5.6-terra", "output_tokens": 42, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:16.097122+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005296, "created_at": "2026-09-17T03:09:16.098Z" }, { "sequence": 117, "attempt_id": "d3b4c0ed-604a-4774-afd6-2c867621ea3a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 1.3221060330979526, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:16.449938+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001876, "created_at": "2026-09-17T03:09:16.450Z" }, { "sequence": 118, "attempt_id": "4aae08ab-8f60-4988-af40-9638874aa5fe", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007298, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.8037703649606556, "model_name": "claude-sonnet-5", "output_tokens": 161, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:16.633366+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007298, "created_at": "2026-09-17T03:09:16.633Z" }, { "sequence": 119, "attempt_id": "8c03119a-e009-4198-8fd8-ac7efd2f1560", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.7736874680267647, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:17.428440+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000374, "created_at": "2026-09-17T03:09:17.428Z" }, { "sequence": 120, "attempt_id": "60d65ad5-8842-483e-9006-d752012f2d0a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.5171551459934562, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:17.614657+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004976000000000001, "created_at": "2026-09-17T03:09:17.614Z" }, { "sequence": 121, "attempt_id": "650a65c8-eebe-4ef1-87d6-fc9fd528c93b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.122300901100971, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:17.799435+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02474, "created_at": "2026-09-17T03:09:17.799Z" }, { "sequence": 122, "attempt_id": "034ae485-23c4-4590-be65-b7e5daf123bf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011096, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.8864609780721366, "model_name": "gpt-5.6-sol", "output_tokens": 74, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:17.878590+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011096, "created_at": "2026-09-17T03:09:17.878Z" }, { "sequence": 123, "attempt_id": "882a5508-8e48-4862-9275-221823b4f270", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00572, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.065890035009943, "model_name": "gpt-5.6-terra", "output_tokens": 76, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:18.156011+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00572, "created_at": "2026-09-17T03:09:18.156Z" }, { "sequence": 124, "attempt_id": "d4372bcd-b05f-4e3a-97c2-f1f22e53a8b4", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00310125, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 2.319943635025993, "model_name": "gemini-3.8-flash", "output_tokens": 596, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:09:18.789949+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00310125, "created_at": "2026-09-17T03:09:18.790Z" }, { "sequence": 125, "attempt_id": "d9470aa6-2ec8-407e-8de7-09775db65673", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.6956140589900315, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:09:19.505964+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:09:19.506Z" }, { "sequence": 126, "attempt_id": "b166ce0b-202b-4681-9694-0a30a765a8b7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01270225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 12.001281325006858, "model_name": "muse-spark-1.2", "output_tokens": 2222, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:19.750902+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01270225, "created_at": "2026-09-17T03:09:19.751Z" }, { "sequence": 127, "attempt_id": "11324dbb-d328-4d34-9a2b-e86ad5a4bb9a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.6353563100565225, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:19.811651+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T03:09:19.811Z" }, { "sequence": 128, "attempt_id": "feba0868-61ef-46cf-952c-cfd209619b1c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.0523949339985847, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:19.950966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T03:09:19.951Z" }, { "sequence": 129, "attempt_id": "af94c773-5c15-48a6-b418-31d394c903e9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1523244089912623, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:09:21.124099+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:09:21.124Z" }, { "sequence": 130, "attempt_id": "c7dc9a3c-f11f-423d-9bd0-0aee494b79c8", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.837596982019022, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:21.363960+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:09:21.364Z" }, { "sequence": 131, "attempt_id": "1a6bcc59-5ef5-49a1-9142-523fc7e901a9", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008586, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 5.478442589985207, "model_name": "gemini-3.1-pro-preview", "output_tokens": 523, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:21.596292+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008586, "created_at": "2026-09-17T03:09:21.596Z" }, { "sequence": 132, "attempt_id": "96a2ca55-2700-45d7-ab1e-ce5897e9b05b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011872, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.881188826984726, "model_name": "gpt-5.6-sol", "output_tokens": 110, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:22.652638+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011872, "created_at": "2026-09-17T03:09:22.652Z" }, { "sequence": 133, "attempt_id": "9c41fd24-2bf0-4059-8344-24a6185dcb70", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.062646355945617, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:22.679517+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036389999999999995, "created_at": "2026-09-17T03:09:22.679Z" }, { "sequence": 134, "attempt_id": "f133073a-ae99-4146-baad-a9fdfc903d88", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 5.704538463032804, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:23.523932+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:09:23.524Z" }, { "sequence": 135, "attempt_id": "90b248aa-3a89-489a-aaf0-ffc1d3164574", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00235725, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.2525211159372702, "model_name": "gemini-3.8-flash", "output_tokens": 396, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:23.637022+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00235725, "created_at": "2026-09-17T03:09:23.637Z" }, { "sequence": 136, "attempt_id": "3bcbf9ab-ca34-4699-bf74-54f9cdcfba87", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007836, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 4.791155445971526, "model_name": "gpt-5.6-terra", "output_tokens": 250, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:24.623093+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007836, "created_at": "2026-09-17T03:09:24.623Z" }, { "sequence": 137, "attempt_id": "5094e50c-1ce1-4b0b-90d8-a7377e5e0bf9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01076675, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 22.374490175046958, "model_name": "muse-spark-1.3", "output_tokens": 1786, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:25.929573+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01076675, "created_at": "2026-09-17T03:09:25.929Z" }, { "sequence": 138, "attempt_id": "1d4b12ca-9442-4a92-8122-0d2df25c9a23", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016055, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.64645134692546, "model_name": "claude-opus-5", "output_tokens": 77, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:26.304028+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016055, "created_at": "2026-09-17T03:09:26.304Z" }, { "sequence": 139, "attempt_id": "f269d8cf-60eb-47bd-a973-7915f253fefe", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006582, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.9724495799746364, "model_name": "claude-sonnet-5", "output_tokens": 93, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:26.619042+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006582, "created_at": "2026-09-17T03:09:26.619Z" }, { "sequence": 140, "attempt_id": "34a46cfc-e4a9-49a1-9dba-a5517ef4652f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 1.0519587270682678, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:27.000305+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001856, "created_at": "2026-09-17T03:09:27.000Z" }, { "sequence": 141, "attempt_id": "25abd4fb-cdca-4876-8b47-7ea9633eac3c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0112275, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 19.424858235986903, "model_name": "muse-spark-1.3", "output_tokens": 1875, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:27.103321+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0112275, "created_at": "2026-09-17T03:09:27.103Z" }, { "sequence": 142, "attempt_id": "3be9e574-dca3-438f-a306-cbd86b076839", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008278, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 6.0640768519369885, "model_name": "gemini-3.1-pro-preview", "output_tokens": 496, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:27.217082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008278, "created_at": "2026-09-17T03:09:27.217Z" }, { "sequence": 143, "attempt_id": "0bf3d72a-c237-4d75-9e1a-06af0e1b6689", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7022918849252164, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:28.026123+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02454, "created_at": "2026-09-17T03:09:28.026Z" }, { "sequence": 144, "attempt_id": "8b4298fe-1883-4ad5-8e01-ab51e93ef260", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02928, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.721293470007367, "model_name": "claude-fable-5-1", "output_tokens": 20, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:28.265668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02928, "created_at": "2026-09-17T03:09:28.265Z" }, { "sequence": 145, "attempt_id": "5592a969-511f-4dbd-b8d5-c13f03dcd7cc", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004936000000000001, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.166809018002823, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:28.293162+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004936000000000001, "created_at": "2026-09-17T03:09:28.293Z" }, { "sequence": 146, "attempt_id": "6a233a25-3fbc-45b0-87d4-614a07b0f9f0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005416, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.6031076459912583, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:28.622424+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005416, "created_at": "2026-09-17T03:09:28.622Z" }, { "sequence": 147, "attempt_id": "d211d298-0726-495d-8f36-8161feb0f839", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.83835165691562, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:29.131850+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003668, "created_at": "2026-09-17T03:09:29.132Z" }, { "sequence": 148, "attempt_id": "c1a59663-5644-4af8-acc4-fc0a9602adb2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010836, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.8488486980786547, "model_name": "gpt-5.6-sol", "output_tokens": 65, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:29.486560+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010836, "created_at": "2026-09-17T03:09:29.486Z" }, { "sequence": 149, "attempt_id": "4025f6cd-ccae-4169-b968-2ac2345234a1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00858725, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 12.823870418942533, "model_name": "muse-spark-1.3", "output_tokens": 1272, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:30.272571+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00858725, "created_at": "2026-09-17T03:09:30.272Z" }, { "sequence": 150, "attempt_id": "2ba644c6-d66c-4242-aa41-fef15f7601f3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01358525, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 13.174600254977122, "model_name": "muse-spark-1.2", "output_tokens": 2448, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:30.809553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01358525, "created_at": "2026-09-17T03:09:30.809Z" }, { "sequence": 151, "attempt_id": "b5a61a40-73a1-4a65-8f57-52ce52f89004", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.2101724159438163, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:31.715917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:09:31.716Z" }, { "sequence": 152, "attempt_id": "dd369286-c844-47cb-8251-c8bf4423789f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.1159568969160318, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:31.948190+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:09:31.948Z" }, { "sequence": 153, "attempt_id": "7f9694b7-12e8-4130-87a0-9b54321f8229", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.8654500189004466, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:32.157474+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:09:32.157Z" }, { "sequence": 154, "attempt_id": "fc053485-b09d-44bf-bfba-915aa9e5a39d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.489006040035747, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:32.639693+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:09:32.639Z" }, { "sequence": 155, "attempt_id": "1a18dae2-c6e9-4565-b9fc-897f1e06339f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9731998980278149, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:33.150957+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:09:33.151Z" }, { "sequence": 156, "attempt_id": "262de69a-a89c-4877-8362-17e707dce025", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.4117417819797993, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:33.383679+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:09:33.383Z" }, { "sequence": 157, "attempt_id": "379b6c8c-fd81-4f2d-8192-5aefad58107e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.8574320260668173, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:33.528874+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:09:33.529Z" }, { "sequence": 158, "attempt_id": "3094b63d-c431-4cb6-83ba-6a544a7e8647", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.2433836930431426, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:33.986786+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:09:33.986Z" }, { "sequence": 159, "attempt_id": "aceab2b6-2222-41a1-8890-74fdb6066266", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.106446776073426, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:34.665982+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:09:34.666Z" }, { "sequence": 160, "attempt_id": "e1a1adf0-bacc-4b83-b2c9-fa0b87a9c0c4", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00127275, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.4899919169256464, "model_name": "gemini-3.8-flash", "output_tokens": 112, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:34.905132+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00127275, "created_at": "2026-09-17T03:09:34.905Z" }, { "sequence": 161, "attempt_id": "1a988e8f-25bd-4282-b0ad-1655861ddc71", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0034845, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 6.851408962043934, "model_name": "gemini-3.8-flash", "output_tokens": 703, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:34.925750+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0034845, "created_at": "2026-09-17T03:09:34.925Z" }, { "sequence": 162, "attempt_id": "f80aecb0-a725-4b5a-940b-2ba42dccaf90", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0092515, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 7.568004612927325, "model_name": "muse-spark-1.2", "output_tokens": 1433, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:36.213755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0092515, "created_at": "2026-09-17T03:09:36.213Z" }, { "sequence": 163, "attempt_id": "a6d7c91d-eaa5-4032-af96-40d19973da67", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003438, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.1597562370589003, "model_name": "gemini-3.1-pro-preview", "output_tokens": 97, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:36.341518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003438, "created_at": "2026-09-17T03:09:36.341Z" }, { "sequence": 164, "attempt_id": "f595de9d-764d-4ba3-908c-ec8340027a66", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012966, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 9.6022658420261, "model_name": "gemini-3.1-pro-preview", "output_tokens": 892, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:36.843860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012966, "created_at": "2026-09-17T03:09:36.844Z" }, { "sequence": 165, "attempt_id": "e5971b98-bfa1-4347-b68b-7f5f4f4d3737", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004647, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.31621462397743, "model_name": "muse-spark-1.2", "output_tokens": 349, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:37.002380+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004647, "created_at": "2026-09-17T03:09:37.002Z" }, { "sequence": 166, "attempt_id": "1af16c4d-7085-48f2-b93e-6ae8dbe49a82", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001871, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.692226946935989, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:37.053278+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001871, "created_at": "2026-09-17T03:09:37.053Z" }, { "sequence": 167, "attempt_id": "9dcd413d-3d08-4184-b7eb-49dd7d1aade2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00500825, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.5806955749867484, "model_name": "muse-spark-1.3", "output_tokens": 434, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:37.587692+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00500825, "created_at": "2026-09-17T03:09:37.587Z" }, { "sequence": 168, "attempt_id": "0704986f-02cd-408e-9a00-2c02e62ac3a1", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016295, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.975410785060376, "model_name": "claude-opus-5", "output_tokens": 83, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:37.920482+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016295, "created_at": "2026-09-17T03:09:37.920Z" }, { "sequence": 169, "attempt_id": "6d09a730-8b45-4849-a21c-5c8bb581eff4", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 1.9920707419514656, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:38.225816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005798, "created_at": "2026-09-17T03:09:38.225Z" }, { "sequence": 170, "attempt_id": "157285eb-cd0d-4442-9a91-3367486862c6", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 3.568294532946311, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:38.494377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02901, "created_at": "2026-09-17T03:09:38.494Z" }, { "sequence": 171, "attempt_id": "f16db085-9ae4-46b5-96ec-75585541ccc1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005564, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.534426738973707, "model_name": "gpt-5.6-terra", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:38.608232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005564, "created_at": "2026-09-17T03:09:38.608Z" }, { "sequence": 172, "attempt_id": "dd809f10-a864-4171-b0fb-c219d375d752", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.0697989619802684, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:38.934657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02474, "created_at": "2026-09-17T03:09:38.934Z" }, { "sequence": 173, "attempt_id": "020ecdf2-f6a7-4663-b8a8-b8379971a376", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.114373376010917, "model_name": "gpt-5.6-sol", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:39.136985+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011056, "created_at": "2026-09-17T03:09:39.137Z" }, { "sequence": 174, "attempt_id": "280fb016-3101-4146-9ca0-3627d0a2a6f7", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.8688003710703924, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:09:39.383863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000374, "created_at": "2026-09-17T03:09:39.384Z" }, { "sequence": 175, "attempt_id": "fe617297-6366-4aba-802d-0ec4335cf600", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00195375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.5565504420083016, "model_name": "gemini-3.8-flash", "output_tokens": 290, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:09:39.802908+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00195375, "created_at": "2026-09-17T03:09:39.803Z" }, { "sequence": 176, "attempt_id": "2cc4d8c6-9c09-42c4-af09-582d0808d879", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.293215571087785, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:39.901076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004976000000000001, "created_at": "2026-09-17T03:09:39.901Z" }, { "sequence": 177, "attempt_id": "5bfde5e3-5965-4222-911f-1d7171b72f89", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6765057180309668, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:40.615535+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T03:09:40.615Z" }, { "sequence": 178, "attempt_id": "fe170910-5580-4abc-9040-775ce9d17cd7", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.097016524989158, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:41.501267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T03:09:41.501Z" }, { "sequence": 179, "attempt_id": "4b609717-96fe-4f92-b901-22b135e24df5", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.8659404340432957, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:41.689477+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:09:41.689Z" }, { "sequence": 180, "attempt_id": "664995d6-638c-4f3e-87f4-d74a7ee22429", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.6377637879922986, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:42.273976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:09:42.274Z" }, { "sequence": 181, "attempt_id": "7fc4845e-0502-4cea-a2ce-26124be92f6c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0985863619716838, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:42.620433+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:09:42.620Z" }, { "sequence": 182, "attempt_id": "ccd12125-4b44-4a99-ae0f-335abd52caab", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006822, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 4.862825941992924, "model_name": "gemini-3.1-pro-preview", "output_tokens": 376, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:42.804318+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006822, "created_at": "2026-09-17T03:09:42.804Z" }, { "sequence": 183, "attempt_id": "5cef2e15-daf4-4468-843c-f308eea8c9f1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.1162295179674402, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:42.826442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:09:42.826Z" }, { "sequence": 184, "attempt_id": "a10e8330-a0f0-4fdf-8309-1de175b03a1f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.9625481580151245, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:43.120362+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T03:09:43.120Z" }, { "sequence": 185, "attempt_id": "a39f0289-09f7-490c-b180-3335a5fe4b10", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.0463262939592823, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:43.891776+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:09:43.891Z" }, { "sequence": 186, "attempt_id": "2672105c-bd9a-4264-bb7f-997709424325", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00970625, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 15.617339341086335, "model_name": "muse-spark-1.3", "output_tokens": 1540, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:43.930386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00970625, "created_at": "2026-09-17T03:09:43.930Z" }, { "sequence": 187, "attempt_id": "09a27120-5916-4807-9141-14b9d66bb4e8", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00131175, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.5039367449935526, "model_name": "gemini-3.8-flash", "output_tokens": 127, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:44.328157+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00131175, "created_at": "2026-09-17T03:09:44.328Z" }, { "sequence": 188, "attempt_id": "5c4dc469-329c-434c-bf94-b08709fb17d8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054465, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 6.438414352014661, "model_name": "muse-spark-1.3", "output_tokens": 533, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:45.066709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0054465, "created_at": "2026-09-17T03:09:45.066Z" }, { "sequence": 189, "attempt_id": "1c6b623b-4059-496e-ac89-dc4caf53e1f4", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00392125, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.2428099979879335, "model_name": "muse-spark-1.2", "output_tokens": 185, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:45.156370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00392125, "created_at": "2026-09-17T03:09:45.156Z" }, { "sequence": 190, "attempt_id": "88c988f5-f6da-4701-a77a-540a2e6f96ff", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 3.464021837920882, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:45.764917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:09:45.765Z" }, { "sequence": 191, "attempt_id": "44ca03c7-feab-458b-b911-1c9b84c5362c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.082425906904973, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:46.432090+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T03:09:46.432Z" }, { "sequence": 192, "attempt_id": "c613dc10-9cc4-412f-af83-99f443fe159e", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003932, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.835171901038848, "model_name": "gemini-3.1-pro-preview", "output_tokens": 142, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:46.476561+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003932, "created_at": "2026-09-17T03:09:46.476Z" }, { "sequence": 193, "attempt_id": "9db14bfe-67a5-46dc-8a47-b6d8327f28fb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00849375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 7.659704041900113, "model_name": "muse-spark-1.2", "output_tokens": 1250, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:46.615040+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00849375, "created_at": "2026-09-17T03:09:46.615Z" }, { "sequence": 194, "attempt_id": "294d5b7a-e659-4fef-b497-bb5bf0d46a05", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00458, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.743313462007791, "model_name": "muse-spark-1.3", "output_tokens": 340, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:46.884907+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00458, "created_at": "2026-09-17T03:09:46.885Z" }, { "sequence": 195, "attempt_id": "2462e76b-038e-420b-9950-26d718b88d6c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.042702225036919, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:47.130523+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T03:09:47.130Z" }, { "sequence": 196, "attempt_id": "e8917004-4ef7-495e-b145-05de66191473", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.5985565350856632, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:47.384993+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:09:47.385Z" }, { "sequence": 197, "attempt_id": "7a46faec-0e61-4ece-9aed-d6e875418e51", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1186010940000415, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:47.572467+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:09:47.572Z" }, { "sequence": 198, "attempt_id": "b97aa521-2439-4724-881c-ed263c1627b9", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.6827032970031723, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:47.633268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T03:09:47.633Z" }, { "sequence": 199, "attempt_id": "3369c081-e89d-4009-aed8-8ccb53e0b8f8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1972656630678102, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:47.694840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:09:47.694Z" }, { "sequence": 200, "attempt_id": "726499e7-38f9-4046-96b8-613013238cfe", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.093729438027367, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:47.730536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:09:47.730Z" }, { "sequence": 201, "attempt_id": "b9033b21-46a5-40a0-981e-9dd3f5eaa754", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.9022914699744433, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:48.308955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:09:48.309Z" }, { "sequence": 202, "attempt_id": "05700bc1-7c7d-4fbf-a41b-e9facfd0c2d4", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02550275, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 26.348972603096627, "model_name": "muse-spark-1.2", "output_tokens": 5248, "prediction": { "choice": "A" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:49.047799+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02550275, "created_at": "2026-09-17T03:09:49.047Z" }, { "sequence": 203, "attempt_id": "1f96798e-cac8-438b-a414-5aaaa0188872", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01316075, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 26.64600040204823, "model_name": "muse-spark-1.3", "output_tokens": 2344, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:49.319899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01316075, "created_at": "2026-09-17T03:09:49.320Z" }, { "sequence": 204, "attempt_id": "cda5d1e4-4d4d-4fa5-a6f3-7e6bbe053d6e", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.5498646770138294, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:49.620097+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:09:49.620Z" }, { "sequence": 205, "attempt_id": "a4a171f6-9c04-4574-b7a1-b6a5a3d4c4a6", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003826, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.900731120025739, "model_name": "gemini-3.1-pro-preview", "output_tokens": 133, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:49.806867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003826, "created_at": "2026-09-17T03:09:49.807Z" }, { "sequence": 206, "attempt_id": "ea697977-f094-491f-815e-4195c83af95f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9576499729882926, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:50.288458+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:09:50.288Z" }, { "sequence": 207, "attempt_id": "434a86ee-db69-49c7-b052-7a84c77a4507", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00428125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.66416120599024, "model_name": "muse-spark-1.2", "output_tokens": 270, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:50.318936+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00428125, "created_at": "2026-09-17T03:09:50.319Z" }, { "sequence": 208, "attempt_id": "17d3ddc9-c758-4d25-bafa-a866ad414139", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0048125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.974001520080492, "model_name": "muse-spark-1.3", "output_tokens": 395, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:50.567926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0048125, "created_at": "2026-09-17T03:09:50.568Z" }, { "sequence": 209, "attempt_id": "26c5ac38-ebdc-40fe-876a-56f7a19ca7ad", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.6178258300060406, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:50.959372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:09:50.959Z" }, { "sequence": 210, "attempt_id": "6c0c2bc1-dcce-417e-87f6-38f3df1526e2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.3832455329829827, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:51.232178+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:09:51.232Z" }, { "sequence": 211, "attempt_id": "109fec6a-fd0e-49a5-a7d9-644eb6f8a6e7", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.531889377045445, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:51.254988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:09:51.256Z" }, { "sequence": 212, "attempt_id": "f0fa0204-667d-4574-bce8-567b0f62b537", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9005800540326163, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:51.544554+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:09:51.544Z" }, { "sequence": 213, "attempt_id": "19953d9c-485e-4295-9492-36a26d52f294", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7972928069066256, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:51.779457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:09:51.779Z" }, { "sequence": 214, "attempt_id": "05a9d549-2bdb-48a8-8917-f38ba6ea5e54", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012705, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.3400554630206898, "model_name": "gemini-3.8-flash", "output_tokens": 112, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:51.929705+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0012705, "created_at": "2026-09-17T03:09:51.929Z" }, { "sequence": 215, "attempt_id": "4a256544-a2ee-4e89-a4ad-e557e0a11deb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9257008320419118, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:52.236917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:09:52.237Z" }, { "sequence": 216, "attempt_id": "cf842224-5a7e-4423-b06a-1c1adb2588cc", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00364025, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.322725179954432, "model_name": "muse-spark-1.3", "output_tokens": 113, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:52.578038+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00364025, "created_at": "2026-09-17T03:09:52.578Z" }, { "sequence": 217, "attempt_id": "cd34d6b5-c9d8-4e96-a812-b23154e4e0b6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 7.462108353967778, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:52.639394+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T03:09:52.639Z" }, { "sequence": 218, "attempt_id": "07927b8d-0829-493d-bd8b-6fc51a859edd", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.8017545890761539, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:53.065247+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T03:09:53.065Z" }, { "sequence": 219, "attempt_id": "c7731ea9-a243-40bb-8e31-4a9ca2fcfb58", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004129, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.9000874490011483, "model_name": "muse-spark-1.2", "output_tokens": 228, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:53.176634+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004129, "created_at": "2026-09-17T03:09:53.176Z" }, { "sequence": 220, "attempt_id": "b3733e0d-554c-4cfd-9451-6605de7fec1a", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003696, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.0935737270629033, "model_name": "gemini-3.1-pro-preview", "output_tokens": 119, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:53.436727+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003696, "created_at": "2026-09-17T03:09:53.436Z" }, { "sequence": 221, "attempt_id": "b6713524-f018-4fd1-bac6-a076fb7f36d7", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.7750401130178943, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:53.731535+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005736, "created_at": "2026-09-17T03:09:53.731Z" }, { "sequence": 222, "attempt_id": "4e2c0a4d-7d34-4d18-9907-d89f381a53e8", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.0556223950115964, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:53.857000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01434, "created_at": "2026-09-17T03:09:53.857Z" }, { "sequence": 223, "attempt_id": "32fcb481-a4c2-483c-a802-dbed32e0650e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.2993307060096413, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:53.964653+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:09:53.964Z" }, { "sequence": 224, "attempt_id": "9a1cde7a-165d-483e-b6d6-f6a85b2fe8ab", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.6752977590076625, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:54.277334+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:09:54.277Z" }, { "sequence": 225, "attempt_id": "95878c21-5fca-4398-8895-51e58908bf2f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005398, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.4586686909897253, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:54.547752+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005398, "created_at": "2026-09-17T03:09:54.547Z" }, { "sequence": 226, "attempt_id": "6f32498f-c8eb-4fde-b6a9-4789b3f62957", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.8650625739246607, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:54.749351+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:09:54.749Z" }, { "sequence": 227, "attempt_id": "8c48d886-a31d-4e20-957f-c473190f2cc8", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.479193859035149, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:55.045720+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T03:09:55.045Z" }, { "sequence": 228, "attempt_id": "9cd100cf-e20c-4bd6-bb09-4cdc7723bfe8", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00144, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.395345486002043, "model_name": "gemini-3.8-flash", "output_tokens": 156, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:55.150059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00144, "created_at": "2026-09-17T03:09:55.150Z" }, { "sequence": 229, "attempt_id": "c961aa17-81fd-45e4-9063-068a267d4ba6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.073725391062908, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:55.273496+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:09:55.273Z" }, { "sequence": 230, "attempt_id": "a80d24cd-46ab-4b38-bff2-8887feaa1cbe", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.9970288219628856, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:56.170807+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001856, "created_at": "2026-09-17T03:09:56.171Z" }, { "sequence": 231, "attempt_id": "e5a60102-c1d9-4ada-81d9-586bfdeeacf8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004669, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 2.235084339044988, "model_name": "muse-spark-1.2", "output_tokens": 353, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:56.536056+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004669, "created_at": "2026-09-17T03:09:56.536Z" }, { "sequence": 232, "attempt_id": "70236a30-4ee9-41fe-8049-f800796a959f", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.720175395021215, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:57.016512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02454, "created_at": "2026-09-17T03:09:57.016Z" }, { "sequence": 233, "attempt_id": "ea98de3b-2f74-4d99-b941-a18cc27bed4a", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016055, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.4887738339602947, "model_name": "claude-opus-5", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:57.263908+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016055, "created_at": "2026-09-17T03:09:57.264Z" }, { "sequence": 234, "attempt_id": "e87a4efc-0390-4b92-a40a-2ca97825b3d8", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006552, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.4702348400605842, "model_name": "claude-sonnet-5", "output_tokens": 90, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:57.539130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006552, "created_at": "2026-09-17T03:09:57.539Z" }, { "sequence": 235, "attempt_id": "f0aa05f1-b7c2-4b72-8001-90777f4c9782", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005452, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.5035825170343742, "model_name": "gpt-5.6-terra", "output_tokens": 57, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:58.063644+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005452, "created_at": "2026-09-17T03:09:58.063Z" }, { "sequence": 236, "attempt_id": "010fc9be-e50a-4f7b-bfc3-8c9010559271", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010876, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.9372499979799613, "model_name": "gpt-5.6-sol", "output_tokens": 67, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:58.144774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010876, "created_at": "2026-09-17T03:09:58.144Z" }, { "sequence": 237, "attempt_id": "7423952d-f87d-4368-9307-da47fb265ee3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0050855, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.643806732958183, "model_name": "muse-spark-1.3", "output_tokens": 451, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:58.631744+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0050855, "created_at": "2026-09-17T03:09:58.631Z" }, { "sequence": 238, "attempt_id": "dff74d9f-c71a-4a83-8c22-b20c268a3bac", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000562, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7746451859129593, "model_name": "gpt-5.6-luna", "output_tokens": 71, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:58.814170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000562, "created_at": "2026-09-17T03:09:58.814Z" }, { "sequence": 239, "attempt_id": "ccd17b08-df4a-4934-bd8c-d5bb53d0b2df", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003543, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.7992349660489708, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:58.886202+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003543, "created_at": "2026-09-17T03:09:58.886Z" }, { "sequence": 240, "attempt_id": "29bb9397-5fbb-4880-a080-9ad673060205", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02963, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.677318845991977, "model_name": "claude-fable-5-1", "output_tokens": 27, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": " {\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:59.248105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02963, "created_at": "2026-09-17T03:09:59.248Z" }, { "sequence": 241, "attempt_id": "9317cf55-2ea8-475d-8a03-80e99a59ec0c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005136, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 6.060403982060961, "model_name": "gemini-3.1-pro-preview", "output_tokens": 238, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:59.520201+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005136, "created_at": "2026-09-17T03:09:59.520Z" }, { "sequence": 242, "attempt_id": "4875fcb8-cf6a-48bc-99ac-5ea57de43697", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0016125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 12.912153468932956, "model_name": "gemini-3.8-flash", "output_tokens": 207, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:00.064791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0016125, "created_at": "2026-09-17T03:10:00.064Z" }, { "sequence": 243, "attempt_id": "d627be23-ab65-4048-855d-98bcc93f9489", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.798267264966853, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:00.342389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T03:10:00.342Z" }, { "sequence": 244, "attempt_id": "72d4c8b5-99fd-490d-a827-4776e5392a76", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.126288832980208, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:00.400157+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00571, "created_at": "2026-09-17T03:10:00.400Z" }, { "sequence": 245, "attempt_id": "7664c743-7e9a-4923-ba20-b19bf3cb76cd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.3574229090008885, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:01.723694+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:10:01.723Z" }, { "sequence": 246, "attempt_id": "fa1b660a-adec-4442-8df7-fb60da20950e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0053, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.3304282169556245, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:01.754241+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0053, "created_at": "2026-09-17T03:10:01.754Z" }, { "sequence": 247, "attempt_id": "36ebe72b-0502-4ef8-8185-6f2e11cdcc81", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.9162903079995885, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:02.005667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:10:02.005Z" }, { "sequence": 248, "attempt_id": "ef19da46-7816-4933-a3c1-3bb06c7f5219", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00486075, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 4.522623442928307, "model_name": "gemini-3.8-flash", "output_tokens": 1070, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:02.085169+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00486075, "created_at": "2026-09-17T03:10:02.085Z" }, { "sequence": 249, "attempt_id": "4f6a558c-9808-4d6a-818f-dda7e5f80b0a", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 14.475530446041375, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:02.227230+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:10:02.227Z" }, { "sequence": 250, "attempt_id": "fce462dc-8b4d-4537-b932-e9aba861b91b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.58432379597798, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:02.423822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02857, "created_at": "2026-09-17T03:10:02.423Z" }, { "sequence": 251, "attempt_id": "26d539f9-8838-4c03-b855-22a457e8663e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 4.120169869042002, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:03.031582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014275, "created_at": "2026-09-17T03:10:03.031Z" }, { "sequence": 252, "attempt_id": "e3cf3862-94f4-46a1-a280-d1caac9233ad", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.040268320008181, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:03.148726+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:10:03.148Z" }, { "sequence": 253, "attempt_id": "f0563bf8-322b-4f7f-a7c3-de754c9db504", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.405931688961573, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:03.172708+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:10:03.172Z" }, { "sequence": 254, "attempt_id": "6b2818c4-4882-4d2b-9a26-d3181fe25b89", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00146775, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.5656305470038205, "model_name": "gemini-3.8-flash", "output_tokens": 166, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:03.598747+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00146775, "created_at": "2026-09-17T03:10:03.598Z" }, { "sequence": 255, "attempt_id": "fc2a8710-040e-4341-8026-cf1b41ccffd9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003858, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.4006804400123656, "model_name": "muse-spark-1.2", "output_tokens": 166, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:03.848533+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003858, "created_at": "2026-09-17T03:10:03.848Z" }, { "sequence": 256, "attempt_id": "020bcbbe-a4cc-4bee-a786-3efe7522f22b", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 1.2315073180943727, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:04.853700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T03:10:04.853Z" }, { "sequence": 257, "attempt_id": "4ca5e82e-c5ef-4896-a7a8-dda368d767c7", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.7282609860412776, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:04.928427+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:10:04.928Z" }, { "sequence": 258, "attempt_id": "9491331f-ea0e-4884-860c-55e8ffef01a3", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.085148103069514, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:05.258216+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T03:10:05.258Z" }, { "sequence": 259, "attempt_id": "418951bb-a792-41a1-930b-351738e4a717", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 2.400648330920376, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:05.455799+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T03:10:05.455Z" }, { "sequence": 260, "attempt_id": "b1329601-fcc6-46f0-8f1f-e22b8d893ab3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00770025, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 6.876269077067263, "model_name": "muse-spark-1.2", "output_tokens": 1068, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:05.531782+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00770025, "created_at": "2026-09-17T03:10:05.531Z" }, { "sequence": 261, "attempt_id": "912b63dc-569e-4076-bbf8-33711c59c53f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01335, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 8.3414646780584, "model_name": "gemini-3.1-pro-preview", "output_tokens": 924, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:05.629292+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01335, "created_at": "2026-09-17T03:10:05.629Z" }, { "sequence": 262, "attempt_id": "cac87110-3e02-407b-a8d0-fd46d079bdd9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00477175, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 3.8446933729574084, "model_name": "muse-spark-1.3", "output_tokens": 381, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:06.095276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00477175, "created_at": "2026-09-17T03:10:06.095Z" }, { "sequence": 263, "attempt_id": "af5ef259-991d-45c9-afac-5845f7654ce2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.2545208160299808, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:06.208547+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:10:06.208Z" }, { "sequence": 264, "attempt_id": "43b5de1b-5ea3-4373-a7e2-f25803345fed", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.3925387469353154, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:06.269851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:10:06.270Z" }, { "sequence": 265, "attempt_id": "2d114401-0e4a-4a87-b2f1-06df7d28ee13", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006862, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 5.055059200967662, "model_name": "gemini-3.1-pro-preview", "output_tokens": 384, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:06.831686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006862, "created_at": "2026-09-17T03:10:06.831Z" }, { "sequence": 266, "attempt_id": "62beb690-30e5-49c9-8c56-6c9b346a57c8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.5539156720042229, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:06.855778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:10:06.856Z" }, { "sequence": 267, "attempt_id": "0eb67862-e04c-4166-9336-4664a1dbe32f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125175, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.4500604700297117, "model_name": "gemini-3.8-flash", "output_tokens": 111, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:07.009259+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00125175, "created_at": "2026-09-17T03:10:07.009Z" }, { "sequence": 268, "attempt_id": "3d8c2da7-dee0-492f-ab54-b605de7a6fa6", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.5009731630561873, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:07.154011+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:10:07.154Z" }, { "sequence": 269, "attempt_id": "47f632f8-d4a6-4601-8010-1a89e23004a4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 3.9905719599919394, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:07.862785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:10:07.862Z" }, { "sequence": 270, "attempt_id": "f8a7d7ad-1d42-429a-af9e-f57190312325", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00411675, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.762105548987165, "model_name": "muse-spark-1.2", "output_tokens": 231, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:07.995002+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00411675, "created_at": "2026-09-17T03:10:07.995Z" }, { "sequence": 271, "attempt_id": "8b74d8d5-3762-4007-8008-7cc746644c76", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 1.1099543180316687, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:08.157314+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T03:10:08.157Z" }, { "sequence": 272, "attempt_id": "64d42d73-bdd8-4003-8e7a-bc28758c565f", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.7409636679803953, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:08.919689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:10:08.919Z" }, { "sequence": 273, "attempt_id": "f2684cf9-43e2-4b5f-9eb2-e0616369d73a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.014358806074597, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.033842+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:10:09.034Z" }, { "sequence": 274, "attempt_id": "98552743-ba9b-484c-829b-855b0016e045", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.2447374620242044, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.100792+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T03:10:09.100Z" }, { "sequence": 275, "attempt_id": "e5c72776-32eb-40e2-9033-84bb739afb08", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.3271011530887336, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.214192+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:10:09.214Z" }, { "sequence": 276, "attempt_id": "4b9cb3a7-958e-43b2-91d1-3b4a41bed644", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1121679389616475, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.294103+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:10:09.294Z" }, { "sequence": 277, "attempt_id": "48ffb3a3-b9ca-42f4-a0ca-b5d7e3ccb45c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00439725, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.2946902750991285, "model_name": "muse-spark-1.3", "output_tokens": 297, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.413840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00439725, "created_at": "2026-09-17T03:10:09.414Z" }, { "sequence": 278, "attempt_id": "11a0851d-d6bd-40ee-b56c-91062d90539e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.996524781920016, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.877074+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T03:10:09.877Z" }, { "sequence": 279, "attempt_id": "b29e9f4e-8471-4b9d-9f05-00a61fc69238", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.6852307179942727, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:09.983307+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T03:10:09.983Z" }, { "sequence": 280, "attempt_id": "db9768d4-83c8-469b-8d5a-1d08e43da4b6", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0089574550511315, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:10.133886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:10:10.134Z" }, { "sequence": 281, "attempt_id": "1dc60029-975b-41c6-ace5-b4ec1bf2291a", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00608, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 4.734445938025601, "model_name": "gemini-3.1-pro-preview", "output_tokens": 321, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:10.215014+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00608, "created_at": "2026-09-17T03:10:10.215Z" }, { "sequence": 282, "attempt_id": "a9062b57-0d8c-432c-a54d-b15043181b97", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6728745650034398, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:10.833369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:10:10.833Z" }, { "sequence": 283, "attempt_id": "ba5cf04c-05f3-4a2c-90da-8b25b1feb441", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0045235, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.0965032660169527, "model_name": "muse-spark-1.2", "output_tokens": 327, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:11.415083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0045235, "created_at": "2026-09-17T03:10:11.415Z" }, { "sequence": 284, "attempt_id": "2dc84422-146a-4f41-9d60-e26209af7ba6", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00113625, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.35884459502995, "model_name": "gemini-3.8-flash", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:11.441561+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00113625, "created_at": "2026-09-17T03:10:11.441Z" }, { "sequence": 285, "attempt_id": "924bb918-71c7-424c-8333-1180ddc8d216", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7675636460771784, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:11.775772+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:10:11.775Z" }, { "sequence": 286, "attempt_id": "44599fa9-615e-4820-836b-eddd198e911c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9782618869794533, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:11.880719+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:10:11.880Z" }, { "sequence": 287, "attempt_id": "36ab0351-704f-420d-9a98-27c64bd84a71", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008699, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 13.931639526970685, "model_name": "muse-spark-1.3", "output_tokens": 1303, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:12.099324+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008699, "created_at": "2026-09-17T03:10:12.099Z" }, { "sequence": 288, "attempt_id": "499752c8-2b2c-4132-b16a-e54fdc9237e8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.4093011820223182, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:12.268472+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:10:12.269Z" }, { "sequence": 289, "attempt_id": "3c547ef5-b4aa-42eb-a66c-5e637af22649", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.029341942979954, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:12.298287+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:10:12.300Z" }, { "sequence": 290, "attempt_id": "f4bd96dd-c92a-44c1-93a8-b809803b26b7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.833894670009613, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:12.326899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:10:12.327Z" }, { "sequence": 291, "attempt_id": "42326d8d-524a-4817-bff8-57ff22588a63", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0009053819812834, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:12.442796+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:10:12.442Z" }, { "sequence": 292, "attempt_id": "ae5337ca-fe4e-4735-b79c-36ab2511da7e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00431525, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.469685825984925, "model_name": "muse-spark-1.3", "output_tokens": 278, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:12.708650+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00431525, "created_at": "2026-09-17T03:10:12.708Z" }, { "sequence": 293, "attempt_id": "78597714-3b37-49c6-b9f7-e71e778fd988", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.9607002430129796, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:13.085489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:10:13.085Z" }, { "sequence": 294, "attempt_id": "17e8f1da-5245-42dd-b956-0a3c75fedabb", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0011505, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.2859864979982376, "model_name": "gemini-3.8-flash", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:13.191870+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0011505, "created_at": "2026-09-17T03:10:13.192Z" }, { "sequence": 295, "attempt_id": "dab887f2-295c-4a3b-820a-a343fb3dbfa1", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.950038366019726, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:13.388043+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:10:13.388Z" }, { "sequence": 296, "attempt_id": "9b5ed20d-0d94-4853-a9ee-45ffc7f78ed3", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005818, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.456593708018772, "model_name": "gemini-3.1-pro-preview", "output_tokens": 299, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:13.414735+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005818, "created_at": "2026-09-17T03:10:13.414Z" }, { "sequence": 297, "attempt_id": "31eb0eab-c521-4cb1-84e8-0c0b3b4f1cff", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00396325, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.4941320500802249, "model_name": "muse-spark-1.2", "output_tokens": 189, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:13.821407+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00396325, "created_at": "2026-09-17T03:10:13.821Z" }, { "sequence": 298, "attempt_id": "72ecc1f2-8ba7-4748-9409-68c92a7a089b", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 1.0591772539773956, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:14.171751+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T03:10:14.171Z" }, { "sequence": 299, "attempt_id": "b8750109-80b1-4442-8312-904aae6375ca", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.1018430680269375, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:14.516901+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:10:14.517Z" }, { "sequence": 300, "attempt_id": "f4a36a24-2a06-49cb-828e-934c1a846682", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00407375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.389686929062009, "model_name": "muse-spark-1.3", "output_tokens": 215, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:14.688357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00407375, "created_at": "2026-09-17T03:10:14.688Z" }, { "sequence": 301, "attempt_id": "1b35e04a-b7ff-4bac-896b-dd8c853861e5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0053, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.303852560929954, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:14.744019+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0053, "created_at": "2026-09-17T03:10:14.744Z" }, { "sequence": 302, "attempt_id": "8af62e84-e868-4dfb-a3c8-e6d17802c6db", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00613, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.0867390809580684, "model_name": "claude-sonnet-5", "output_tokens": 53, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:14.821234+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00613, "created_at": "2026-09-17T03:10:14.821Z" }, { "sequence": 303, "attempt_id": "c0f38539-b122-4615-bf65-35e72d45be12", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004176, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.349651094060391, "model_name": "gemini-3.1-pro-preview", "output_tokens": 159, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:15.151582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004176, "created_at": "2026-09-17T03:10:15.151Z" }, { "sequence": 304, "attempt_id": "57b9e4e1-d928-43f2-9222-cfa86e280d4f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.6830964089604095, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:15.531069+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:10:15.531Z" }, { "sequence": 305, "attempt_id": "727195c9-fe4f-4c35-acb7-94d41201c2a2", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.469360061106272, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:15.822767+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02857, "created_at": "2026-09-17T03:10:15.822Z" }, { "sequence": 306, "attempt_id": "791fb461-6936-4262-9e24-1072aacdd511", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.3593448960455135, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:16.073916+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:10:16.074Z" }, { "sequence": 307, "attempt_id": "d0a1be1a-b92b-4ae0-9013-dedec5f4e1cc", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017625, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 3.794309615972452, "model_name": "claude-opus-5", "output_tokens": 145, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:16.263079+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017625, "created_at": "2026-09-17T03:10:16.263Z" }, { "sequence": 308, "attempt_id": "d488ff80-a38e-4522-8945-86dfa8f2f64c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 3.2275245611090213, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:16.447533+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:10:16.447Z" }, { "sequence": 309, "attempt_id": "cc088b6f-ecb5-471f-bccb-1536c01ee73a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6572984280064702, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:16.758020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T03:10:16.758Z" }, { "sequence": 310, "attempt_id": "bfe79d68-7fea-479f-83fc-b7a3be03e359", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0026265, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 2.827945190947503, "model_name": "gemini-3.8-flash", "output_tokens": 475, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:17.370915+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0026265, "created_at": "2026-09-17T03:10:17.371Z" }, { "sequence": 311, "attempt_id": "0d215415-4bb2-435c-a306-cf94b3fe5264", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.6203896190272644, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:17.469473+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:10:17.469Z" }, { "sequence": 312, "attempt_id": "4189b1db-4dd5-4247-8010-796357378bc5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.140832717064768, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:17.925537+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:10:17.925Z" }, { "sequence": 313, "attempt_id": "30292728-1062-4d44-915a-782ddbd5c973", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.5393286850303411, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:18.013390+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:10:18.013Z" }, { "sequence": 314, "attempt_id": "3f96402e-7be0-460c-8400-a55d5086204c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.7762822039658204, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:18.073180+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:10:18.073Z" }, { "sequence": 315, "attempt_id": "86de4bf7-41fe-41e5-8ae8-32dbf8d78455", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.8649026489583775, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:18.262772+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:10:18.262Z" }, { "sequence": 316, "attempt_id": "7adc913d-5ef3-4215-b500-1cb04b62a155", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00613, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.4835802969755605, "model_name": "gemini-3.1-pro-preview", "output_tokens": 323, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:18.682493+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00613, "created_at": "2026-09-17T03:10:18.682Z" }, { "sequence": 317, "attempt_id": "ef8f3f23-8254-4c97-9770-2b599658f009", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.6167358650127426, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:18.794400+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T03:10:18.794Z" }, { "sequence": 318, "attempt_id": "b9c81e8a-009d-41fa-ba99-2f9269aa1d7f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.0543175500351936, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:19.095634+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:10:19.095Z" }, { "sequence": 319, "attempt_id": "281740d8-eca4-4fe1-aceb-c0e6774b95fe", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00375125, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.9841872620163485, "model_name": "muse-spark-1.2", "output_tokens": 145, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:20.275083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00375125, "created_at": "2026-09-17T03:10:20.275Z" }, { "sequence": 320, "attempt_id": "9be0dce2-4f8f-4772-b8e1-e3a797621239", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00137925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 2.7004978199256584, "model_name": "gemini-3.8-flash", "output_tokens": 145, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:20.654080+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00137925, "created_at": "2026-09-17T03:10:20.654Z" }, { "sequence": 321, "attempt_id": "c110db81-1172-4ea9-9e90-baa9a97ce84e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.9101696560392156, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:20.764138+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014315, "created_at": "2026-09-17T03:10:20.764Z" }, { "sequence": 322, "attempt_id": "2dda7a39-f4a8-4d60-a11d-015cce0da727", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00540925, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 6.069431961979717, "model_name": "muse-spark-1.3", "output_tokens": 531, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:20.838916+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00540925, "created_at": "2026-09-17T03:10:20.839Z" }, { "sequence": 323, "attempt_id": "7a16ba48-5e53-427e-9cb7-6dbdceb7f7d9", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.7493067380273715, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:21.051287+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T03:10:21.051Z" }, { "sequence": 324, "attempt_id": "7e9c22a4-8775-4f17-b58a-f2838f48a4e9", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.96852619596757, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:21.091363+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T03:10:21.091Z" }, { "sequence": 325, "attempt_id": "093f17ae-99d4-4482-9e8a-35028c696a34", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005396, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.892907712026499, "model_name": "gemini-3.1-pro-preview", "output_tokens": 264, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:21.389579+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005396, "created_at": "2026-09-17T03:10:21.389Z" }, { "sequence": 326, "attempt_id": "b8ab158a-2169-48bd-9113-de0842505956", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.345005262992345, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:22.067943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T03:10:22.068Z" }, { "sequence": 327, "attempt_id": "9f94f1a7-5602-44a6-8ece-d8f01676df02", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2471144960727543, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:22.124342+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:10:22.124Z" }, { "sequence": 328, "attempt_id": "08e20687-b5a8-482b-862e-0f1ae7d310cb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00475, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 4.132129449979402, "model_name": "muse-spark-1.3", "output_tokens": 380, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:22.236471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00475, "created_at": "2026-09-17T03:10:22.236Z" }, { "sequence": 329, "attempt_id": "aa445c56-172b-4653-9e13-2d4c6030476f", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.8799487149808556, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:22.561208+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:10:22.561Z" }, { "sequence": 330, "attempt_id": "1a886f29-19ba-4fdb-b24c-c28d048d4b6d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.5394904139684513, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:22.618881+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:10:22.619Z" }, { "sequence": 331, "attempt_id": "f08bd890-a201-4f22-a1d3-2d76d69a97e7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006918, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 8.089940823963843, "model_name": "muse-spark-1.2", "output_tokens": 886, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:22.940447+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006918, "created_at": "2026-09-17T03:10:22.940Z" }, { "sequence": 332, "attempt_id": "1b819de5-649b-4590-8488-d8c3b3b3e738", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.9104911210015416, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:23.005847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:10:23.005Z" }, { "sequence": 333, "attempt_id": "c1ab20e4-5290-4028-909d-a59f953b790e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.399056599009782, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:23.192150+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:10:23.192Z" }, { "sequence": 334, "attempt_id": "f7b21c2c-5263-4aa2-bc1c-37f4221350a0", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00136275, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.8703401680104434, "model_name": "gemini-3.8-flash", "output_tokens": 136, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:23.287313+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00136275, "created_at": "2026-09-17T03:10:23.287Z" }, { "sequence": 335, "attempt_id": "93d55d3a-1082-4de2-ad84-60e757a60bb8", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.6890128280501813, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:23.721453+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T03:10:23.721Z" }, { "sequence": 336, "attempt_id": "b317be63-740c-4f5e-b98a-4bbc1141cb72", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004069, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.5534410229884088, "model_name": "muse-spark-1.2", "output_tokens": 213, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:23.817157+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004069, "created_at": "2026-09-17T03:10:23.817Z" }, { "sequence": 337, "attempt_id": "6c6d0127-468a-43e9-bdd2-52776bf0b420", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.030497060972266, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:24.347566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:10:24.347Z" }, { "sequence": 338, "attempt_id": "f9e62502-912a-44f9-8e45-10567feea25f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.3878994050901383, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:25.232700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:10:25.232Z" }, { "sequence": 339, "attempt_id": "95dc92ff-0400-4b74-8433-f123c21165e6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.074014561017975, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:25.294188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:10:25.294Z" }, { "sequence": 340, "attempt_id": "d6fba2f2-69c2-4a6a-9e0b-bbecf94d04ef", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005566, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.5915325459791347, "model_name": "gpt-5.6-terra", "output_tokens": 65, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:25.341770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005566, "created_at": "2026-09-17T03:10:25.341Z" }, { "sequence": 341, "attempt_id": "56188c3d-11fa-4195-a5e5-7db32e609131", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006376, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.4541401440510526, "model_name": "claude-sonnet-5", "output_tokens": 75, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:25.442396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006376, "created_at": "2026-09-17T03:10:25.442Z" }, { "sequence": 342, "attempt_id": "01295b23-3a6e-4c98-b5af-362300fcc9e1", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015815, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.9021712449612096, "model_name": "claude-opus-5", "output_tokens": 70, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:25.563817+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015815, "created_at": "2026-09-17T03:10:25.563Z" }, { "sequence": 343, "attempt_id": "6798f673-20b2-4255-8556-9fe81a955998", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 10.261918324045837, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:25.819686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T03:10:25.819Z" }, { "sequence": 344, "attempt_id": "a907d2ed-3865-4649-8976-c259f4498174", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.8878619209863245, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:26.209793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:10:26.209Z" }, { "sequence": 345, "attempt_id": "d07165fd-5f0c-4a79-8e10-83d70f85944a", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007338, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.350948475999758, "model_name": "gemini-3.1-pro-preview", "output_tokens": 422, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:26.468091+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007338, "created_at": "2026-09-17T03:10:26.468Z" }, { "sequence": 346, "attempt_id": "108a4c54-64e3-4668-908a-b6635d255ec0", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00163125, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.2691215500235558, "model_name": "gemini-3.8-flash", "output_tokens": 207, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:26.529554+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00163125, "created_at": "2026-09-17T03:10:26.529Z" }, { "sequence": 347, "attempt_id": "63e289b7-00b5-416e-acf2-d854362586ee", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00483825, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 4.501608905964531, "model_name": "muse-spark-1.3", "output_tokens": 394, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:26.653401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00483825, "created_at": "2026-09-17T03:10:26.653Z" }, { "sequence": 348, "attempt_id": "801c7d71-2e5d-4cb8-ac41-90e6d5a88d3f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6521298569859937, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:27.148458+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:10:27.148Z" }, { "sequence": 349, "attempt_id": "05e2c8f4-46f2-43f1-b621-77a3e615462c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006698, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.471203427063301, "model_name": "claude-sonnet-5", "output_tokens": 102, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:28.709856+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006698, "created_at": "2026-09-17T03:10:28.711Z" }, { "sequence": 350, "attempt_id": "5eb38c68-7e6d-4bcc-95bb-18a8dd248277", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01662, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.8632953109918162, "model_name": "claude-opus-5", "output_tokens": 97, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:28.758379+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01662, "created_at": "2026-09-17T03:10:28.760Z" }, { "sequence": 351, "attempt_id": "9826de80-99a8-4d82-8137-9d6b648374a7", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006348, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.51203180802986, "model_name": "gemini-3.1-pro-preview", "output_tokens": 339, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:28.888100+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006348, "created_at": "2026-09-17T03:10:28.888Z" }, { "sequence": 352, "attempt_id": "82e87a7a-a2a4-434c-b1d6-e80e45dd89a4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.2762004020623863, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:28.957864+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:10:28.958Z" }, { "sequence": 353, "attempt_id": "c8da306e-f6f4-4542-a5a9-072eeb512554", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.421294468920678, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:28.987926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:10:28.988Z" }, { "sequence": 354, "attempt_id": "73c871cf-b7d9-4d2d-8256-63ea0adb9c7c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005652, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.845561953028664, "model_name": "gpt-5.6-terra", "output_tokens": 71, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:29.022156+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005652, "created_at": "2026-09-17T03:10:29.022Z" }, { "sequence": 355, "attempt_id": "47fdad5b-14a6-4bdb-9ec3-b662e41e0607", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00503025, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.004271220066585, "model_name": "muse-spark-1.2", "output_tokens": 438, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:29.474470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00503025, "created_at": "2026-09-17T03:10:29.474Z" }, { "sequence": 356, "attempt_id": "83520d0f-4853-458f-9a16-d3d39de3c9c4", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0050005, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.377719170995988, "model_name": "muse-spark-1.3", "output_tokens": 431, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:29.745733+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0050005, "created_at": "2026-09-17T03:10:29.745Z" }, { "sequence": 357, "attempt_id": "38c5a150-743c-43f3-a3f7-f7be17b7077f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9167336199898273, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:29.905009+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:10:29.905Z" }, { "sequence": 358, "attempt_id": "2cc42c2d-0e57-4984-a6a7-c058fd084607", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 7.499128506984562, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:30.104896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T03:10:30.105Z" }, { "sequence": 359, "attempt_id": "97561b7a-6f6f-43c4-83c2-36676779f0b8", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 4.970326830982231, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:30.562657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:10:30.562Z" }, { "sequence": 360, "attempt_id": "0025e31d-ad00-4380-858b-eabf0b84d32f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005652000000000001, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.899361048010178, "model_name": "gpt-5.6-luna", "output_tokens": 71, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:30.658131+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005652000000000001, "created_at": "2026-09-17T03:10:30.658Z" }, { "sequence": 361, "attempt_id": "60abc20b-a4ec-466f-9c5a-14d190f8dd96", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6923429069574922, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:30.827743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T03:10:30.827Z" }, { "sequence": 362, "attempt_id": "7ece4be4-7101-42f9-81b9-6cd4cba9372d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00241875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.1170510690426454, "model_name": "gemini-3.8-flash", "output_tokens": 415, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:31.034181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00241875, "created_at": "2026-09-17T03:10:31.034Z" }, { "sequence": 363, "attempt_id": "112226d7-2619-4cbf-bea9-2a7b7e43781e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.2775221570627764, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:31.211459+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:10:31.211Z" }, { "sequence": 364, "attempt_id": "3e2f7c6c-9b4e-47a1-824a-c1092ed908a4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0522649630438536, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:31.740596+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:10:31.740Z" }, { "sequence": 365, "attempt_id": "422bbfdf-b1a5-41fb-a7d3-9e4f5d86a4a6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0522027250844985, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:31.911430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:10:31.911Z" }, { "sequence": 366, "attempt_id": "6379cb7e-1ba8-4e21-a928-99e669279382", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.139325671014376, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:31.941487+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T03:10:31.941Z" }, { "sequence": 367, "attempt_id": "0befbf8f-d9dc-45ec-9fb7-76817845e812", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.2331470100907609, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:32.296569+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:10:32.296Z" }, { "sequence": 368, "attempt_id": "ce723dda-ee0a-4220-94e1-4ce5764b73bf", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.8257894199341536, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:32.420351+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:10:32.420Z" }, { "sequence": 369, "attempt_id": "f2efe97a-3deb-443d-b8a8-1a2f9ca9b9c0", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.7055422390112653, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:32.647353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:10:32.647Z" }, { "sequence": 370, "attempt_id": "553570db-6344-4072-946d-5f60682e6309", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.3665910359704867, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:32.870436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T03:10:32.870Z" }, { "sequence": 371, "attempt_id": "0b73f6c6-13f2-4876-a2be-2d7eac9b51bf", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00666125, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 3.8702842589700595, "model_name": "muse-spark-1.2", "output_tokens": 820, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:32.919518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00666125, "created_at": "2026-09-17T03:10:32.919Z" }, { "sequence": 372, "attempt_id": "4215b35e-38a2-44da-a83d-75169a5ddb5f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.18787811009679, "model_name": "gemini-3.8-flash", "output_tokens": 105, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:32.960792+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00122925, "created_at": "2026-09-17T03:10:32.960Z" }, { "sequence": 373, "attempt_id": "5c785fa8-519c-4b89-b81a-c02d1f8cc9c4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006512, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.359754362958483, "model_name": "gemini-3.1-pro-preview", "output_tokens": 351, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:33.151980+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006512, "created_at": "2026-09-17T03:10:33.152Z" }, { "sequence": 374, "attempt_id": "2c8f7c0a-495e-48bb-b91c-8a45efab4a69", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.6650176219409332, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:33.613822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T03:10:33.613Z" }, { "sequence": 375, "attempt_id": "f008dea6-15cd-4548-bf7d-8df98775f89a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00404025, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.5757022839970887, "model_name": "muse-spark-1.2", "output_tokens": 213, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:33.903170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00404025, "created_at": "2026-09-17T03:10:33.903Z" }, { "sequence": 376, "attempt_id": "f68ca7e3-be48-41d4-a1c8-9b10df0008d3", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003668, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.486550014000386, "model_name": "gemini-3.1-pro-preview", "output_tokens": 120, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:34.727703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003668, "created_at": "2026-09-17T03:10:34.727Z" }, { "sequence": 377, "attempt_id": "06957a34-60f3-4765-b6c8-722b60da9662", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.5819555569905788, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:34.763779+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:10:34.763Z" }, { "sequence": 378, "attempt_id": "c234f1f9-b61f-4d3a-87f8-b10268a1fec4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.8348010319750756, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:34.825217+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:10:34.825Z" }, { "sequence": 379, "attempt_id": "ec3801b5-03f4-452a-b908-900129c5be64", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.0262248190119863, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:34.958221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:10:34.958Z" }, { "sequence": 380, "attempt_id": "14b8f42a-e68a-4c6c-9320-874d475a0474", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004495, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.0196947570657358, "model_name": "muse-spark-1.3", "output_tokens": 320, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:34.989761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004495, "created_at": "2026-09-17T03:10:34.989Z" }, { "sequence": 381, "attempt_id": "6eb7e9f5-952c-495a-83f2-171b586b1b74", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005438, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.6337794399587438, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:35.279587+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005438, "created_at": "2026-09-17T03:10:35.279Z" }, { "sequence": 382, "attempt_id": "d9a20640-a88f-4974-b7a3-c89210522692", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006518, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.399390753940679, "model_name": "claude-sonnet-5", "output_tokens": 89, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:35.309045+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006518, "created_at": "2026-09-17T03:10:35.309Z" }, { "sequence": 383, "attempt_id": "1a43630b-a8f8-46df-a1ee-009198cb1fe4", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015495, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.772365016047843, "model_name": "claude-opus-5", "output_tokens": 57, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:35.452730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015495, "created_at": "2026-09-17T03:10:35.452Z" }, { "sequence": 384, "attempt_id": "0ceab312-72d0-49b6-ae02-2462fd2b93c9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.52566443500109, "model_name": "muse-spark-1.3", "output_tokens": 635, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:35.542285+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005875, "created_at": "2026-09-17T03:10:35.542Z" }, { "sequence": 385, "attempt_id": "bb4889bd-db9e-4928-b722-46e1bbf30ccb", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.0621369431028143, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:35.916677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003683, "created_at": "2026-09-17T03:10:35.916Z" }, { "sequence": 386, "attempt_id": "a4196c89-749f-4559-9148-31d7701836a5", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 4.009503436041996, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:36.459955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T03:10:36.460Z" }, { "sequence": 387, "attempt_id": "0fbd1f81-323e-4c6a-a761-08939e609d40", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.25267909897957, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:36.837927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:10:36.838Z" }, { "sequence": 388, "attempt_id": "a59a45e1-c489-4360-b843-bf6233ff837e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00210075, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.09246955497656, "model_name": "gemini-3.8-flash", "output_tokens": 333, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:36.883986+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00210075, "created_at": "2026-09-17T03:10:36.884Z" }, { "sequence": 389, "attempt_id": "322a3276-3ce9-417d-be03-206b24e7a60f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.60539149295073, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:37.087450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:10:37.087Z" }, { "sequence": 390, "attempt_id": "044e26c8-4208-49e2-a315-5fc7fa63cad5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.294426295091398, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:37.635882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:10:37.636Z" }, { "sequence": 391, "attempt_id": "20ceb509-4304-4667-bec5-ca777f9fec27", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00467125, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 2.7067400850355625, "model_name": "muse-spark-1.2", "output_tokens": 355, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:37.730234+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00467125, "created_at": "2026-09-17T03:10:37.730Z" }, { "sequence": 392, "attempt_id": "0856354c-d712-41c3-bfd6-3bee24478e4e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2694896130124107, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:37.760158+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:10:37.760Z" }, { "sequence": 393, "attempt_id": "8d335d2b-3564-4f96-b888-b240992ae02c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0394072469789535, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:37.907379+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:10:37.907Z" }, { "sequence": 394, "attempt_id": "8875fd26-00c8-41f2-bd48-6b9c300cb0ee", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0797593770548701, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:37.992150+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:10:37.992Z" }, { "sequence": 395, "attempt_id": "4abda47a-d0d9-41d2-80d6-188aa7947f2b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.353095152066089, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:38.302174+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:10:38.302Z" }, { "sequence": 396, "attempt_id": "643e6a45-ca88-440a-acc7-92cdaba0f21d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.9498083159560338, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:38.710287+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003561, "created_at": "2026-09-17T03:10:38.710Z" }, { "sequence": 397, "attempt_id": "73ea52fa-0422-42db-8ffa-e3cf78d33bda", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005692, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 3.9606221690773964, "model_name": "gemini-3.1-pro-preview", "output_tokens": 285, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:38.742316+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005692, "created_at": "2026-09-17T03:10:38.742Z" }, { "sequence": 398, "attempt_id": "4c5a3e18-1e33-451c-8d47-832e65e6da22", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00130275, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.0751137239858508, "model_name": "gemini-3.8-flash", "output_tokens": 120, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:38.771942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00130275, "created_at": "2026-09-17T03:10:38.772Z" }, { "sequence": 399, "attempt_id": "7da39f28-229f-4581-abdd-5f8abfceaa93", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.6513717600610107, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:38.960741+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:10:38.960Z" }, { "sequence": 400, "attempt_id": "48bb5e1c-58bf-4105-b15c-a2de2830f5db", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00418375, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.578616491984576, "model_name": "muse-spark-1.2", "output_tokens": 240, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:39.515774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00418375, "created_at": "2026-09-17T03:10:39.515Z" }, { "sequence": 401, "attempt_id": "55ba7698-5603-4f4f-8bd4-2da451462c96", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.9619470369070768, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:39.766771+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001833, "created_at": "2026-09-17T03:10:39.766Z" }, { "sequence": 402, "attempt_id": "1c7cc464-5e26-4f01-bc74-b14d3e76ff77", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00419225, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.6534700450720266, "model_name": "muse-spark-1.3", "output_tokens": 242, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:40.443236+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00419225, "created_at": "2026-09-17T03:10:40.443Z" }, { "sequence": 403, "attempt_id": "ce5cf5ea-cc98-4030-8dbc-3ee9f218a90f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.7761452429695055, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:40.518760+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005684, "created_at": "2026-09-17T03:10:40.518Z" }, { "sequence": 404, "attempt_id": "2655c373-161b-49b3-af8c-7838b890f965", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004062, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.4672219670610502, "model_name": "gemini-3.1-pro-preview", "output_tokens": 149, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:40.585389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004062, "created_at": "2026-09-17T03:10:40.585Z" }, { "sequence": 405, "attempt_id": "3c414ec0-755e-4f45-ac28-9beefd8453c3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.6120838450733572, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:40.614920+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009752, "created_at": "2026-09-17T03:10:40.616Z" }, { "sequence": 406, "attempt_id": "815190e7-b2ac-45c0-82b5-9f41aed3f37f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00533, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 5.7758921650238335, "model_name": "muse-spark-1.3", "output_tokens": 510, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:40.763619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00533, "created_at": "2026-09-17T03:10:40.763Z" }, { "sequence": 407, "attempt_id": "8f4c3286-7b64-4c81-af54-39e08b6f74f6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.2372675898950547, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:40.794817+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004904, "created_at": "2026-09-17T03:10:40.795Z" }, { "sequence": 408, "attempt_id": "dd1736bf-9580-40b1-8f75-4de210a7cf59", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.6272938000038266, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:41.434186+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02438, "created_at": "2026-09-17T03:10:41.434Z" }, { "sequence": 409, "attempt_id": "9c96215a-4ffe-4ee1-be01-18a703016d77", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 3.533167466055602, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:41.555070+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02844, "created_at": "2026-09-17T03:10:41.555Z" }, { "sequence": 410, "attempt_id": "3fa67e96-03d0-4f10-80f5-9bfef3c6f7e2", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0523491769563407, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:41.667615+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:10:41.667Z" }, { "sequence": 411, "attempt_id": "40880a98-0c69-4ebf-9f15-d44756173b25", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.9579902840778232, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:41.763356+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004904, "created_at": "2026-09-17T03:10:41.763Z" }, { "sequence": 412, "attempt_id": "250082ed-83a8-4677-813c-9d96b61f5e39", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00115125, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.4303931050235406, "model_name": "gemini-3.8-flash", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:41.979998+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00115125, "created_at": "2026-09-17T03:10:41.980Z" }, { "sequence": 413, "attempt_id": "5b44e7f0-bbd3-4c15-92a6-ce96962d1c18", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0037015, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.3908638369757682, "model_name": "muse-spark-1.3", "output_tokens": 133, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:42.036161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0037015, "created_at": "2026-09-17T03:10:42.036Z" }, { "sequence": 414, "attempt_id": "f18bfe69-ad28-4282-a651-2098f66ec87c", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.679461546940729, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:42.378544+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:10:42.378Z" }, { "sequence": 415, "attempt_id": "fede20b6-2751-44da-82c2-fe93bf7de6a1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004067, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.7994764050235972, "model_name": "muse-spark-1.2", "output_tokens": 219, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:42.594610+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004067, "created_at": "2026-09-17T03:10:42.594Z" }, { "sequence": 416, "attempt_id": "264bab0e-0753-4cc4-ac4e-ccd3a0ad6021", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9653506799368188, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:43.032050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:10:43.032Z" }, { "sequence": 417, "attempt_id": "aa9ff456-df17-4118-b86d-9f5103a5839c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1865640809992328, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:43.201358+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:10:43.201Z" }, { "sequence": 418, "attempt_id": "61ddf655-a2cd-4b0c-8087-b0ca41379ba6", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.7404538140399382, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:43.331443+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:10:43.331Z" }, { "sequence": 419, "attempt_id": "069d2b15-4c16-41a0-9811-eb136ea3d820", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 5.496777446940541, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:43.829177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01421, "created_at": "2026-09-17T03:10:43.829Z" }, { "sequence": 420, "attempt_id": "46621f9c-95d8-449a-8533-612f57332967", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.8065449090208858, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:44.039007+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00035009999999999994, "created_at": "2026-09-17T03:10:44.039Z" }, { "sequence": 421, "attempt_id": "e5eb65db-f65d-415c-9736-5e1d27c49c0b", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0013065, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.0491720760473981, "model_name": "gemini-3.8-flash", "output_tokens": 125, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:44.113627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0013065, "created_at": "2026-09-17T03:10:44.113Z" }, { "sequence": 422, "attempt_id": "84d9d7ac-7f2b-41e3-8a63-198600ef9ccf", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.6924663690151647, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:44.159515+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:10:44.159Z" }, { "sequence": 423, "attempt_id": "19fb8e03-54c5-4bd4-b27d-727ae42a6b45", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.769145073951222, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:44.189434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:10:44.191Z" }, { "sequence": 424, "attempt_id": "c6c2cbdb-1286-44e4-8325-2f57d9393001", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.5257802330888808, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:44.354436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:10:44.354Z" }, { "sequence": 425, "attempt_id": "a1b7acd1-a18b-4374-a704-3731dcd9b405", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6025357840117067, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:44.823594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:10:44.823Z" }, { "sequence": 426, "attempt_id": "b5e7426d-c08d-4851-ab61-86648e84646f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004906, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.542901128064841, "model_name": "gemini-3.1-pro-preview", "output_tokens": 223, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:45.016158+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004906, "created_at": "2026-09-17T03:10:45.016Z" }, { "sequence": 427, "attempt_id": "aff94c06-b757-4de3-bf4e-3b87ae37edac", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.264407465001568, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:45.058868+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:10:45.058Z" }, { "sequence": 428, "attempt_id": "a6b75850-8297-462f-9558-fb7c73303ef4", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.3241011969512329, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:45.513871+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:10:45.514Z" }, { "sequence": 429, "attempt_id": "5ab07a06-ee04-4b67-a4f4-52411f7b1493", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040355, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.7472971200477332, "model_name": "muse-spark-1.2", "output_tokens": 211, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:45.610378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0040355, "created_at": "2026-09-17T03:10:45.610Z" }, { "sequence": 430, "attempt_id": "89a31d0e-f5c8-4eea-a288-4991da80c254", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7133158709621057, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:46.107625+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:10:46.107Z" }, { "sequence": 431, "attempt_id": "5b1bd90b-67fc-4a42-aac7-1fd74ea58995", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.093614288023673, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:46.238127+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:10:46.238Z" }, { "sequence": 432, "attempt_id": "937c9103-ea22-4353-84b3-f843b24fb2f8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.4651514800498262, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:46.325130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:10:46.325Z" }, { "sequence": 433, "attempt_id": "0a43252b-98df-40da-9e24-2783fb1c5ada", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00508, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.35691913892515, "model_name": "gpt-5.6-terra", "output_tokens": 25, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:46.404568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00508, "created_at": "2026-09-17T03:10:46.404Z" }, { "sequence": 434, "attempt_id": "39466e6c-3de9-4b66-9dd3-f6422d8499e2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00462625, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.2013260659296066, "model_name": "muse-spark-1.3", "output_tokens": 350, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:46.565890+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00462625, "created_at": "2026-09-17T03:10:46.566Z" }, { "sequence": 435, "attempt_id": "158901ee-284e-40ab-98a3-a566a7d9311f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003758, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 4.114626103080809, "model_name": "gemini-3.1-pro-preview", "output_tokens": 127, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:46.741440+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003758, "created_at": "2026-09-17T03:10:46.741Z" }, { "sequence": 436, "attempt_id": "eba8798c-7eb4-4195-bcd5-3ce971f63c53", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.846882791025564, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:46.935541+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:10:46.935Z" }, { "sequence": 437, "attempt_id": "3ed7ac82-ccf3-4ec0-a333-11e7f0a56dbc", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.2158444750821218, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:47.362306+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003561, "created_at": "2026-09-17T03:10:47.362Z" }, { "sequence": 438, "attempt_id": "98cd9d97-18a4-4b00-a5c1-f6758a0bda93", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7028968119993806, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:47.678828+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:10:47.679Z" }, { "sequence": 439, "attempt_id": "209f4188-1c25-471e-a5f5-1849dbd5def3", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.7396499280584976, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:47.810101+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:10:47.810Z" }, { "sequence": 440, "attempt_id": "d081b300-43f2-4cb7-95ab-3c93c4569289", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001239, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.2208751519210637, "model_name": "gemini-3.8-flash", "output_tokens": 103, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:47.869793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001239, "created_at": "2026-09-17T03:10:47.869Z" }, { "sequence": 441, "attempt_id": "bad4e428-f718-4d42-9cd5-f923a624f0e3", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003378, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.9457292270381004, "model_name": "gemini-3.1-pro-preview", "output_tokens": 92, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:48.499209+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003378, "created_at": "2026-09-17T03:10:48.499Z" }, { "sequence": 442, "attempt_id": "9eabc0f0-a1d0-49eb-a9fb-7b11f7c7583d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00415825, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.3348857139935717, "model_name": "muse-spark-1.3", "output_tokens": 234, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:48.611398+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00415825, "created_at": "2026-09-17T03:10:48.611Z" }, { "sequence": 443, "attempt_id": "b5746f6a-2d31-41df-a953-9af5c715082d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00466825, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.522034148918465, "model_name": "muse-spark-1.2", "output_tokens": 354, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:48.892475+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00466825, "created_at": "2026-09-17T03:10:48.892Z" }, { "sequence": 444, "attempt_id": "2cf6d30f-87ed-4078-a8a8-bbc054cb2d10", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006628, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.271789849968627, "model_name": "claude-sonnet-5", "output_tokens": 76, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:49.054795+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006628, "created_at": "2026-09-17T03:10:49.054Z" }, { "sequence": 445, "attempt_id": "149f6c54-d011-49a0-88bd-bb6b410dcdd7", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016095, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.617702503106557, "model_name": "claude-opus-5", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:49.227391+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016095, "created_at": "2026-09-17T03:10:49.227Z" }, { "sequence": 446, "attempt_id": "5502152e-54aa-42cb-be98-1a36cd96c0e7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011564, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.0113980500027537, "model_name": "gpt-5.6-sol", "output_tokens": 85, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:49.730941+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011564, "created_at": "2026-09-17T03:10:49.731Z" }, { "sequence": 447, "attempt_id": "e57a9189-3862-40e1-aff3-9ed62b8d181c", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8732684089336544, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:49.806063+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003789, "created_at": "2026-09-17T03:10:49.806Z" }, { "sequence": 448, "attempt_id": "b2fae74a-3aba-4828-8a36-0d99086e1298", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.4300462240353227, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:49.849594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:10:49.851Z" }, { "sequence": 449, "attempt_id": "c10632ea-4cd8-4f9c-9d82-5c4fa3dbbb35", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.5814688249956816, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:50.025365+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:10:50.025Z" }, { "sequence": 450, "attempt_id": "a937be35-f093-430a-925b-c587392b8c9b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005808, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.1766321070026606, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:50.060713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005808, "created_at": "2026-09-17T03:10:50.062Z" }, { "sequence": 451, "attempt_id": "c7aa2ced-a82a-44f5-a0f7-67c046a8b0ed", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00200475, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.036297575919889, "model_name": "gemini-3.8-flash", "output_tokens": 292, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:50.687954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00200475, "created_at": "2026-09-17T03:10:50.688Z" }, { "sequence": 452, "attempt_id": "4b3a4cc9-08c1-4594-a9cc-1e40e3a53eb1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.6525632579578087, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:50.728492+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T03:10:50.728Z" }, { "sequence": 453, "attempt_id": "3e77fc83-1b2e-438a-976f-4a6ccfa0bc01", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000618, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.1086905639385805, "model_name": "gpt-5.6-luna", "output_tokens": 104, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:51.018509+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000618, "created_at": "2026-09-17T03:10:51.018Z" }, { "sequence": 454, "attempt_id": "d8c69a29-a47e-48f0-80ae-47d5e2172a38", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.0707319510402158, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:51.799608+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:10:51.799Z" }, { "sequence": 455, "attempt_id": "38e845ea-e1e7-4560-b76c-a3a48eabe496", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.0697016679914668, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:52.130085+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:10:52.130Z" }, { "sequence": 456, "attempt_id": "009ecb4e-81de-4b52-b150-9ab3d9ef71f7", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01552, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.3478611019672826, "model_name": "claude-opus-5", "output_tokens": 58, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:52.197968+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01552, "created_at": "2026-09-17T03:10:52.198Z" }, { "sequence": 457, "attempt_id": "778967a1-0e5c-4eb9-909a-1ad3ebc7cb9c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006478, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.3738485400099307, "model_name": "claude-sonnet-5", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:52.265538+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006478, "created_at": "2026-09-17T03:10:52.265Z" }, { "sequence": 458, "attempt_id": "ce117321-d21b-47b0-9d08-8127feee9896", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.4367002960061654, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:52.530566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:10:52.530Z" }, { "sequence": 459, "attempt_id": "13a6d559-b118-43f1-af33-966feedee1a9", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.1501159869367257, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:52.921880+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T03:10:52.922Z" }, { "sequence": 460, "attempt_id": "6819decc-5158-4113-94da-6e6d3eb6dd8d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00521, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.182305386988446, "model_name": "gpt-5.6-terra", "output_tokens": 36, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:52.964130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00521, "created_at": "2026-09-17T03:10:52.964Z" }, { "sequence": 461, "attempt_id": "041ded65-b41c-4eb2-abf8-7a9366184426", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.7668445679591969, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:53.003845+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003683, "created_at": "2026-09-17T03:10:53.007Z" }, { "sequence": 462, "attempt_id": "c5e2258a-4362-4db2-bb04-347c372bc534", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00223575, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.4124497760785744, "model_name": "gemini-3.8-flash", "output_tokens": 369, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:54.583323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00223575, "created_at": "2026-09-17T03:10:54.583Z" }, { "sequence": 463, "attempt_id": "2111c171-43b4-457a-9173-978be921bf0c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007454, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.250315281911753, "model_name": "gemini-3.1-pro-preview", "output_tokens": 419, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:54.790460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007454, "created_at": "2026-09-17T03:10:54.790Z" }, { "sequence": 464, "attempt_id": "40edad89-62cb-4001-8724-0cdfe6a3cb9f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.816521006054245, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:54.862479+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005736, "created_at": "2026-09-17T03:10:54.862Z" }, { "sequence": 465, "attempt_id": "c985006f-54eb-4462-96e3-16959ff83271", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.9480436759768054, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:54.952195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01434, "created_at": "2026-09-17T03:10:54.952Z" }, { "sequence": 466, "attempt_id": "bbdca172-37b9-4458-a332-122d39386ad9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004888, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 2.4598877739626914, "model_name": "muse-spark-1.2", "output_tokens": 406, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:55.032519+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004888, "created_at": "2026-09-17T03:10:55.032Z" }, { "sequence": 467, "attempt_id": "16760126-33e9-4589-8f0f-6f86a6023ccc", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.6687635401031002, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:55.293378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T03:10:55.293Z" }, { "sequence": 468, "attempt_id": "6fddecce-fe2c-4782-ae0b-04b263bb1e04", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.326130390050821, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:56.290617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T03:10:56.290Z" }, { "sequence": 469, "attempt_id": "4aa9f408-03eb-47ea-80a3-fc8174429ff4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.6039373760577291, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:56.507849+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:10:56.508Z" }, { "sequence": 470, "attempt_id": "ac15f4c9-44d5-4227-ae3e-e649dd75d364", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005194, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.8016148869646713, "model_name": "gpt-5.6-terra", "output_tokens": 34, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:56.796999+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005194, "created_at": "2026-09-17T03:10:56.797Z" }, { "sequence": 471, "attempt_id": "fdf876a1-aeaf-47e4-88da-75e7dc22f75b", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006532, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 5.149035249953158, "model_name": "gemini-3.1-pro-preview", "output_tokens": 355, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:56.990706+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006532, "created_at": "2026-09-17T03:10:56.990Z" }, { "sequence": 472, "attempt_id": "528bbbce-f291-4bcb-b460-2ad21c8b6817", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00900475, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 7.724259817041457, "model_name": "muse-spark-1.2", "output_tokens": 1352, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:57.024577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00900475, "created_at": "2026-09-17T03:10:57.024Z" }, { "sequence": 473, "attempt_id": "a56a6701-cace-46ed-8334-f81f922a2e75", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.363896321039647, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:57.195915+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:10:57.196Z" }, { "sequence": 474, "attempt_id": "823847fa-3e47-41f9-8f46-ab867072108f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00517275, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 5.054415164981037, "model_name": "muse-spark-1.3", "output_tokens": 473, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:57.357854+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00517275, "created_at": "2026-09-17T03:10:57.358Z" }, { "sequence": 475, "attempt_id": "2ee970cb-98a9-458f-b492-4a7b629bf301", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.508151096990332, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:58.059051+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:10:58.059Z" }, { "sequence": 476, "attempt_id": "5e1be89c-7492-4986-b6fe-1d477670134e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 3.026083300006576, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:58.101194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:10:58.102Z" }, { "sequence": 477, "attempt_id": "77f98d19-3ea2-4dc8-b1df-d0f1a7c282b8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003768, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.340015136054717, "model_name": "gemini-3.1-pro-preview", "output_tokens": 124, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:58.675600+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003768, "created_at": "2026-09-17T03:10:58.675Z" }, { "sequence": 478, "attempt_id": "da30fc66-76da-4ef9-ae9e-73bad3f2e4d0", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6548922980437055, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:58.756384+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T03:10:58.756Z" }, { "sequence": 479, "attempt_id": "e97cd292-f18e-454c-8a03-31890b02d083", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.8090167680056766, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:59.209161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00601, "created_at": "2026-09-17T03:10:59.209Z" }, { "sequence": 480, "attempt_id": "f3b82202-4066-49d5-b5c6-31c065461d46", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00719425, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.3903076070128, "model_name": "muse-spark-1.3", "output_tokens": 926, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:59.485681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00719425, "created_at": "2026-09-17T03:10:59.485Z" }, { "sequence": 481, "attempt_id": "3c5b2c30-342a-45ba-84b5-38b103f275d0", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.655574228032492, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:59.799214+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:10:59.799Z" }, { "sequence": 482, "attempt_id": "ab17a4b3-e568-42c6-9307-a812e4e00c68", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041845, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 2.8020781310042366, "model_name": "muse-spark-1.2", "output_tokens": 239, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:59.848988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0041845, "created_at": "2026-09-17T03:10:59.849Z" }, { "sequence": 483, "attempt_id": "b3a16a84-84c6-43c3-a750-03bf93c1735f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1906026049982756, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:59.912290+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:10:59.912Z" }, { "sequence": 484, "attempt_id": "7394c32c-bf8e-4b4b-81b7-da786f91478a", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129375, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.7608423399506137, "model_name": "gemini-3.8-flash", "output_tokens": 117, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:00.094192+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00129375, "created_at": "2026-09-17T03:11:00.094Z" }, { "sequence": 485, "attempt_id": "170bdd2b-164c-40cb-9580-6007a855547a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004635, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.281325545045547, "model_name": "muse-spark-1.3", "output_tokens": 345, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:00.138630+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004635, "created_at": "2026-09-17T03:11:00.140Z" }, { "sequence": 486, "attempt_id": "0315dbbc-ef69-46e8-a774-528cde78c169", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00549, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.5396177229704335, "model_name": "gpt-5.6-terra", "output_tokens": 47, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:00.354238+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00549, "created_at": "2026-09-17T03:11:00.354Z" }, { "sequence": 487, "attempt_id": "e2435769-2569-4c07-acd1-8e54ae67ae58", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.7606249729869887, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:00.652926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:11:00.653Z" }, { "sequence": 488, "attempt_id": "05e36af1-4e67-403f-8fc9-47fc2f03f0fa", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018175, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.8496161659713835, "model_name": "claude-opus-5", "output_tokens": 137, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:01.090106+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.018175, "created_at": "2026-09-17T03:11:01.090Z" }, { "sequence": 489, "attempt_id": "570d34ad-ac7d-41f8-8d3c-e11c08ed3e9c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.323817390948534, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:01.381647+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:11:01.381Z" }, { "sequence": 490, "attempt_id": "61d236ee-55a8-48cc-8eb3-c625b8254791", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000573, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.55294010497164, "model_name": "gpt-5.6-luna", "output_tokens": 67, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:01.798279+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000573, "created_at": "2026-09-17T03:11:01.798Z" }, { "sequence": 491, "attempt_id": "0836ff7d-8640-4405-8486-f66922cc7601", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0016575, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.9831654580775648, "model_name": "gemini-3.8-flash", "output_tokens": 200, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:01.833514+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0016575, "created_at": "2026-09-17T03:11:01.833Z" }, { "sequence": 492, "attempt_id": "1f7bfcc0-e462-4c63-b337-23a3311f10a9", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.7180613810196519, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:01.867850+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001856, "created_at": "2026-09-17T03:11:01.869Z" }, { "sequence": 493, "attempt_id": "0120f62b-2572-47bc-8aef-b77e087ee661", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00525375, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.7834733520867303, "model_name": "muse-spark-1.2", "output_tokens": 470, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:02.922364+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00525375, "created_at": "2026-09-17T03:11:02.922Z" }, { "sequence": 494, "attempt_id": "f8b6f834-0110-4f27-8fb7-6f1dc6724bf2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004936000000000001, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.0226207140367478, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:02.964614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004936000000000001, "created_at": "2026-09-17T03:11:02.964Z" }, { "sequence": 495, "attempt_id": "04c2e938-a9c6-4e97-95b1-863511c13072", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01618, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.698219929006882, "model_name": "claude-opus-5", "output_tokens": 82, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:03.095170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01618, "created_at": "2026-09-17T03:11:03.095Z" }, { "sequence": 496, "attempt_id": "57e56392-8dd3-44c8-9a05-4a13e0384342", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00506, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.622840900090523, "model_name": "gemini-3.1-pro-preview", "output_tokens": 220, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:03.152701+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00506, "created_at": "2026-09-17T03:11:03.152Z" }, { "sequence": 497, "attempt_id": "db9e6c60-a5e5-4a05-a02c-c33dfbe0b716", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.8923140919068828, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:03.318304+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02454, "created_at": "2026-09-17T03:11:03.318Z" }, { "sequence": 498, "attempt_id": "cd2ec0ce-a900-43bf-9208-cd42ee7d9ac4", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006792, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.805858619045466, "model_name": "claude-sonnet-5", "output_tokens": 114, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:03.501678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006792, "created_at": "2026-09-17T03:11:03.501Z" }, { "sequence": 499, "attempt_id": "908206db-cd0c-4300-ade4-cd153ebba179", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00544, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7988968379795551, "model_name": "gpt-5.6-terra", "output_tokens": 56, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:03.667078+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00544, "created_at": "2026-09-17T03:11:03.667Z" }, { "sequence": 500, "attempt_id": "1e42f0f0-f0ac-44c6-9e95-2e05962c60fe", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010956, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.094589365995489, "model_name": "gpt-5.6-sol", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:03.928452+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010956, "created_at": "2026-09-17T03:11:03.928Z" }, { "sequence": 501, "attempt_id": "980e149b-de2e-46c6-8c1e-ed685e6bbc96", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.9614038040162995, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:04.114441+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003668, "created_at": "2026-09-17T03:11:04.114Z" }, { "sequence": 502, "attempt_id": "733a5db0-7010-46c0-b464-bad75c0473bc", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0050455, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.347017379943281, "model_name": "muse-spark-1.3", "output_tokens": 421, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:04.300489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0050455, "created_at": "2026-09-17T03:11:04.300Z" }, { "sequence": 503, "attempt_id": "e0eb856f-b470-4494-abc1-4a52cc1709ff", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.6171250379411504, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:04.767642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T03:11:04.767Z" }, { "sequence": 504, "attempt_id": "5058af52-cc32-454b-bf1d-7cb060b32645", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02918, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.716419636970386, "model_name": "claude-fable-5-1", "output_tokens": 18, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:04.898775+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02918, "created_at": "2026-09-17T03:11:04.898Z" }, { "sequence": 505, "attempt_id": "4acdd719-dbbf-4520-a91a-a197febe72b6", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006018, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 1.5136526779970154, "model_name": "claude-sonnet-5", "output_tokens": 40, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:05.483909+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006018, "created_at": "2026-09-17T03:11:05.484Z" }, { "sequence": 506, "attempt_id": "0682761b-df4b-4dfa-a781-743573cd7397", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.4533678960287943, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:06.388882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005436, "created_at": "2026-09-17T03:11:06.389Z" }, { "sequence": 507, "attempt_id": "d9ff5deb-7560-447f-acf7-3e3656748de2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010652, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.6688925770577043, "model_name": "gpt-5.6-sol", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:06.473360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010652, "created_at": "2026-09-17T03:11:06.473Z" }, { "sequence": 508, "attempt_id": "52de23f9-3823-4ae2-aee5-0c9d91bbc628", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.1463881579693407, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:06.511069+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:11:06.511Z" }, { "sequence": 509, "attempt_id": "c5dd92d6-679e-473d-98c7-50b86cca7c82", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01537, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.8876289980253205, "model_name": "claude-opus-5", "output_tokens": 53, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:06.597392+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01537, "created_at": "2026-09-17T03:11:06.597Z" }, { "sequence": 510, "attempt_id": "9e7fc76d-4f66-433e-a21c-53cb3b5efce6", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00432825, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 3.8784820300061256, "model_name": "gemini-3.8-flash", "output_tokens": 928, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:11:06.885381+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00432825, "created_at": "2026-09-17T03:11:06.885Z" }, { "sequence": 511, "attempt_id": "eab0dad7-7220-4672-b521-14d2af59d0c9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.1412842309800908, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:07.688859+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003555, "created_at": "2026-09-17T03:11:07.689Z" }, { "sequence": 512, "attempt_id": "67ffadb9-57c9-428d-a168-cbe3ab302374", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 5.115354005014524, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:08.652953+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T03:11:08.653Z" }, { "sequence": 513, "attempt_id": "94885de2-2291-4269-b3d0-b1913489637c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005412, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 3.629758076975122, "model_name": "gpt-5.6-luna", "output_tokens": 53, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:09.150052+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005412, "created_at": "2026-09-17T03:11:09.150Z" }, { "sequence": 514, "attempt_id": "91ab10e2-96c3-4e30-9a1f-7f89455f98f6", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.9540644839871675, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:10.642934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:11:10.643Z" }, { "sequence": 515, "attempt_id": "f9ba72db-8ca7-40eb-aaa9-52a8609a1fad", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011034, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 7.931975774932653, "model_name": "gemini-3.1-pro-preview", "output_tokens": 731, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:10.896931+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011034, "created_at": "2026-09-17T03:11:10.897Z" }, { "sequence": 516, "attempt_id": "176ace15-e243-4bb8-8840-870049bcaf2e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.8977587610716, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:11.082862+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:11:11.083Z" }, { "sequence": 517, "attempt_id": "5781b2a8-6b26-4337-99b8-84d75b2039f2", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6711997829843313, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:11.350181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:11:11.350Z" }, { "sequence": 518, "attempt_id": "8fde3d9f-1b70-451a-bc11-df3f1093e18f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00289875, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 4.966433132067323, "model_name": "gemini-3.8-flash", "output_tokens": 546, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:11.477826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00289875, "created_at": "2026-09-17T03:11:11.478Z" }, { "sequence": 519, "attempt_id": "1ca81b02-83cd-457c-94e9-319f1efec74c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.303390273009427, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:12.421609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:11:12.421Z" }, { "sequence": 520, "attempt_id": "95305ffa-0360-4d39-b6f5-446d89b82ce3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1550613860599697, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:12.542313+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:11:12.542Z" }, { "sequence": 521, "attempt_id": "f4a89b1d-fc6f-4824-a5bb-be7749ed3d92", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0290358880301937, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:12.577103+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:11:12.577Z" }, { "sequence": 522, "attempt_id": "61258106-0781-4f41-b4b3-a0840ba000c6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.0246825329959393, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:12.956236+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:11:12.956Z" }, { "sequence": 523, "attempt_id": "732750be-6292-4a43-9f9a-c2b9a11950f1", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.5426894560223445, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:13.268494+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:11:13.268Z" }, { "sequence": 524, "attempt_id": "f28a1009-c171-4d63-a7d7-1bf5113f2bc7", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009266, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 7.297288144007325, "model_name": "gemini-3.1-pro-preview", "output_tokens": 583, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:13.729443+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009266, "created_at": "2026-09-17T03:11:13.729Z" }, { "sequence": 525, "attempt_id": "db82b70f-389b-4963-abd6-b7938b9d00b6", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.1878381250426173, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:13.799239+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:11:13.799Z" }, { "sequence": 526, "attempt_id": "563dc782-029d-4bf7-8cc3-30d434e7fdf4", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00137025, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.6983473630389199, "model_name": "gemini-3.8-flash", "output_tokens": 138, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:14.275739+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00137025, "created_at": "2026-09-17T03:11:14.275Z" }, { "sequence": 527, "attempt_id": "ef231dad-e419-4838-9264-bfe8a3e6d0a8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00756425, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 12.641721123014577, "model_name": "muse-spark-1.3", "output_tokens": 1036, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:15.836460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00756425, "created_at": "2026-09-17T03:11:15.836Z" }, { "sequence": 528, "attempt_id": "f1b4ea3b-1ef2-4cf6-84cc-417e9806aded", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.8590241610072553, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:16.176802+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00571, "created_at": "2026-09-17T03:11:16.176Z" }, { "sequence": 529, "attempt_id": "b3433c37-b071-4df3-8c0c-3b34126c4fdf", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01246025, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 13.156258863047697, "model_name": "muse-spark-1.2", "output_tokens": 2188, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:16.518748+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01246025, "created_at": "2026-09-17T03:11:16.518Z" }, { "sequence": 530, "attempt_id": "e71dd619-66b1-4069-abf7-0003701c8680", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.6820974289439619, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:16.564212+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T03:11:16.565Z" }, { "sequence": 531, "attempt_id": "045e17c8-7992-4a6a-908a-31c5acdfae9a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004698, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.630960409063846, "model_name": "muse-spark-1.3", "output_tokens": 361, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:16.622465+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004698, "created_at": "2026-09-17T03:11:16.622Z" }, { "sequence": 532, "attempt_id": "8cad2438-b8bb-40c5-9143-92d138dcc341", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.8345704849343747, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:16.671459+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014275, "created_at": "2026-09-17T03:11:16.671Z" }, { "sequence": 533, "attempt_id": "fbb0dee0-3413-4e4f-abc6-1f1a767d3189", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.452184319961816, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:17.217568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02857, "created_at": "2026-09-17T03:11:17.217Z" }, { "sequence": 534, "attempt_id": "a4293963-ff0d-458b-ab08-b8cae4d3a290", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.1277307739946991, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:17.692274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:11:17.692Z" }, { "sequence": 535, "attempt_id": "52a53039-9b4e-4158-9e01-2f6158912f88", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004974, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.395406601019204, "model_name": "gemini-3.1-pro-preview", "output_tokens": 225, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:17.852193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004974, "created_at": "2026-09-17T03:11:17.852Z" }, { "sequence": 536, "attempt_id": "e0b183e2-224e-491e-95c0-f9d237dbc874", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005288, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.3855339279398322, "model_name": "gpt-5.6-terra", "output_tokens": 44, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:17.994796+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005288, "created_at": "2026-09-17T03:11:17.994Z" }, { "sequence": 537, "attempt_id": "173eeddf-8dc9-4bfb-9c50-73e4e1b61a08", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.881093348027207, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:18.107518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:11:18.107Z" }, { "sequence": 538, "attempt_id": "60864c46-14bb-4d21-a695-dc8b6ea0b252", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.8570030750706792, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:18.522639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:11:18.522Z" }, { "sequence": 539, "attempt_id": "3b8ddbe2-9a8b-4cda-aa24-87bf949768be", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8774744580732659, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:18.618635+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:11:18.618Z" }, { "sequence": 540, "attempt_id": "71b9b2eb-9dc1-46c6-8b1c-e93e9ad7f321", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044175, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 5.431506837019697, "model_name": "muse-spark-1.2", "output_tokens": 295, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:18.735594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0044175, "created_at": "2026-09-17T03:11:18.735Z" }, { "sequence": 541, "attempt_id": "e436400b-8b4e-4e46-8916-4f20e0cffc9f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01092175, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 11.991850486025214, "model_name": "muse-spark-1.2", "output_tokens": 1826, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:18.913754+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01092175, "created_at": "2026-09-17T03:11:18.913Z" }, { "sequence": 542, "attempt_id": "40d089e1-2b3f-404f-b365-aa4fcb139b0f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00150525, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 2.258630452910438, "model_name": "gemini-3.8-flash", "output_tokens": 176, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:19.521500+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00150525, "created_at": "2026-09-17T03:11:19.521Z" }, { "sequence": 543, "attempt_id": "405350cc-5f6e-470a-8f57-10d215b12b76", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00394725, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.4965054800268263, "model_name": "muse-spark-1.2", "output_tokens": 187, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:19.567728+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00394725, "created_at": "2026-09-17T03:11:19.567Z" }, { "sequence": 544, "attempt_id": "aac07c5a-9ad1-4077-b6c0-c1b6b7b1db94", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.3417717290576547, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:20.004541+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:11:20.004Z" }, { "sequence": 545, "attempt_id": "f9b4ac78-3b3d-441a-a3af-7bd5b0ceafd5", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.4261613889830187, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:20.206000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:11:20.206Z" }, { "sequence": 546, "attempt_id": "7667f55d-7d59-4b24-a84a-5e74561afd4a", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.8355668989242986, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:20.402687+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:11:20.402Z" }, { "sequence": 547, "attempt_id": "b49d4bce-1100-45a2-b996-6a026c6d3ca2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0810740330489352, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:20.649168+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:11:20.649Z" }, { "sequence": 548, "attempt_id": "a42d4136-269d-4bbd-b11a-b066d8e592f3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0769307899754494, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:20.694930+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:11:20.695Z" }, { "sequence": 549, "attempt_id": "9ed3d3a5-b611-4b71-b244-c497c2ee6b3d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7805279919411987, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:20.746599+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:11:20.746Z" }, { "sequence": 550, "attempt_id": "7dd73a31-45a5-4043-ab4a-b1c1c6886735", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9810057999566197, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:21.027265+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:11:21.027Z" }, { "sequence": 551, "attempt_id": "b3532934-6e28-4728-afdf-b6ae0c591943", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00535, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.392102210898884, "model_name": "gemini-3.1-pro-preview", "output_tokens": 258, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:21.105375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00535, "created_at": "2026-09-17T03:11:21.105Z" }, { "sequence": 552, "attempt_id": "47afd9ef-1eb3-47a8-b615-2d0253b0f314", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044615, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 3.4980593159561977, "model_name": "muse-spark-1.3", "output_tokens": 308, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:21.395363+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0044615, "created_at": "2026-09-17T03:11:21.395Z" }, { "sequence": 553, "attempt_id": "79c5c9bd-e819-4763-901e-d8a86c5cf6f8", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7938476919662207, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:21.489061+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:11:21.489Z" }, { "sequence": 554, "attempt_id": "ff3d806e-c3c1-4a5b-b3cf-2a76f5041bf8", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001218, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.3893025440629572, "model_name": "gemini-3.8-flash", "output_tokens": 98, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:21.837706+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001218, "created_at": "2026-09-17T03:11:21.837Z" }, { "sequence": 555, "attempt_id": "27b5f89e-7e08-4deb-935d-9e2bd9f0a4a1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6843972780043259, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:22.217454+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:11:22.217Z" }, { "sequence": 556, "attempt_id": "b23e16e5-9750-4bcb-ad20-4e197ef4c8f9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00455825, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.833311936003156, "model_name": "muse-spark-1.2", "output_tokens": 329, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:22.623438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00455825, "created_at": "2026-09-17T03:11:22.623Z" }, { "sequence": 557, "attempt_id": "64ea3531-1160-4ef4-ab54-b61af5e14a2c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.608392923953943, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:23.048954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:11:23.049Z" }, { "sequence": 558, "attempt_id": "b18c09a5-f1ad-4566-a919-3f2f8d6c8ac1", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9772532409988344, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:23.129058+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:11:23.129Z" }, { "sequence": 559, "attempt_id": "34660e76-67b6-46de-ab39-b08b8dd8d39f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 5.054691047058441, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:23.217175+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:11:23.217Z" }, { "sequence": 560, "attempt_id": "2b26621d-b8e4-4652-ae5b-c258f8089188", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1055986660066992, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:23.369354+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:11:23.369Z" }, { "sequence": 561, "attempt_id": "ee984ffe-a4b1-41be-85f6-0240043a9983", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.5877959999488667, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:23.471229+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:11:23.471Z" }, { "sequence": 562, "attempt_id": "8ebcde8f-43f5-4c36-97e3-b8cfbdb54987", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0042395, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.840854838024825, "model_name": "muse-spark-1.3", "output_tokens": 254, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:23.580915+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0042395, "created_at": "2026-09-17T03:11:23.581Z" }, { "sequence": 563, "attempt_id": "5800b1fd-540e-4fb5-819a-78ed8515d4ec", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003516, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.3816705220378935, "model_name": "gemini-3.1-pro-preview", "output_tokens": 104, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:23.633172+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003516, "created_at": "2026-09-17T03:11:23.633Z" }, { "sequence": 564, "attempt_id": "79e9135c-ad90-4519-b99b-4d4658659ff5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9728773260721937, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:23.682657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:11:23.682Z" }, { "sequence": 565, "attempt_id": "021f8724-9f8c-4f1f-84c1-c28d3f5360a1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9255244820378721, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:24.021274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:11:24.021Z" }, { "sequence": 566, "attempt_id": "1d30d8ee-6d7a-4608-8450-3c7e69771ccf", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.5479685169411823, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:24.621847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:11:24.622Z" }, { "sequence": 567, "attempt_id": "1c1cbda9-26d3-4d60-807b-fa6bf7d07832", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.5045402290998027, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:24.919386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:11:24.919Z" }, { "sequence": 568, "attempt_id": "a7d849e9-79c6-4e02-bcc3-8150645d7162", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0011505, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.1516041950089857, "model_name": "gemini-3.8-flash", "output_tokens": 80, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:25.418062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0011505, "created_at": "2026-09-17T03:11:25.418Z" }, { "sequence": 569, "attempt_id": "9dfcdf46-58b7-4796-b10f-bbe1297a048e", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.8520750760799274, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:25.520065+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001884, "created_at": "2026-09-17T03:11:25.520Z" }, { "sequence": 570, "attempt_id": "eca7e76e-cd53-44a8-a7ee-d3b5733c0868", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004384, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.9339128630235791, "model_name": "muse-spark-1.2", "output_tokens": 288, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:25.562355+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004384, "created_at": "2026-09-17T03:11:25.565Z" }, { "sequence": 571, "attempt_id": "ee5b7b04-32fd-4c1b-b077-ef95e8ab90f6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6342774219810963, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:26.596583+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02484, "created_at": "2026-09-17T03:11:26.596Z" }, { "sequence": 572, "attempt_id": "b89174b4-8cc6-4b5f-ab7b-d5f0e6bc5efc", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006876, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 2.6878300880780444, "model_name": "claude-sonnet-5", "output_tokens": 114, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:26.755259+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006876, "created_at": "2026-09-17T03:11:26.755Z" }, { "sequence": 573, "attempt_id": "1aee0de5-859f-403d-befb-928336351d6f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003396, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.673526880913414, "model_name": "gemini-3.1-pro-preview", "output_tokens": 94, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:26.846777+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003396, "created_at": "2026-09-17T03:11:26.846Z" }, { "sequence": 574, "attempt_id": "7743e248-177b-420a-a502-858e1c4b69c6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00454975, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.5950313940411434, "model_name": "muse-spark-1.3", "output_tokens": 327, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:27.110919+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00454975, "created_at": "2026-09-17T03:11:27.111Z" }, { "sequence": 575, "attempt_id": "def7e569-aca2-48dc-8156-5d39f70b0f22", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00562, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6463626580080017, "model_name": "gpt-5.6-terra", "output_tokens": 66, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:27.209064+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00562, "created_at": "2026-09-17T03:11:27.209Z" }, { "sequence": 576, "attempt_id": "84987354-4186-4d44-abad-bd81ab4d5f7b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010996, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.7791223189560696, "model_name": "gpt-5.6-sol", "output_tokens": 67, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:27.258159+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010996, "created_at": "2026-09-17T03:11:27.260Z" }, { "sequence": 577, "attempt_id": "0cdd22d0-a48b-4192-a60d-8d09172d93c0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000568, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6858870410360396, "model_name": "gpt-5.6-luna", "output_tokens": 71, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:27.310285+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000568, "created_at": "2026-09-17T03:11:27.310Z" }, { "sequence": 578, "attempt_id": "eb6c7fc0-a01a-4f06-9003-d93dc40b2ff9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 1.3650587480515242, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:28.257961+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003761, "created_at": "2026-09-17T03:11:28.258Z" }, { "sequence": 579, "attempt_id": "61a532eb-4281-40a9-bbf1-28b5b229f794", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017665, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 4.977174016064964, "model_name": "claude-opus-5", "output_tokens": 133, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:28.706004+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017665, "created_at": "2026-09-17T03:11:28.706Z" }, { "sequence": 580, "attempt_id": "877b2d04-b3f1-44a6-94fb-30742fbc1bc5", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02965, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 5.771281318971887, "model_name": "claude-fable-5-1", "output_tokens": 19, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:29.454354+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02965, "created_at": "2026-09-17T03:11:29.454Z" }, { "sequence": 581, "attempt_id": "f3b642f8-366d-4f0c-bf8f-b1bcdcc84f8e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002889, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 3.088643671013415, "model_name": "gemini-3.8-flash", "output_tokens": 538, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:29.889360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002889, "created_at": "2026-09-17T03:11:29.889Z" }, { "sequence": 582, "attempt_id": "1d5cb89f-19c6-443f-a075-cdf7fab9cf6b", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.3341470110462978, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:30.086529+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:11:30.086Z" }, { "sequence": 583, "attempt_id": "5115d4c4-e0db-41ce-996c-2d5a78560b0d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.059214233071543, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:11:31.369951+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:11:31.370Z" }, { "sequence": 584, "attempt_id": "4f32aa80-f0ec-41e9-817d-ed7c54bdc25c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00668, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 4.758484165999107, "model_name": "gemini-3.1-pro-preview", "output_tokens": 363, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:31.418983+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00668, "created_at": "2026-09-17T03:11:31.419Z" }, { "sequence": 585, "attempt_id": "c05a4c11-e7ee-4c22-8fd6-abb7bf3a91fd", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.5518138529732823, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:32.059917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:11:32.060Z" }, { "sequence": 586, "attempt_id": "f501d62e-473e-4837-b75f-2d33c2ade6e5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02057, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.25184005103074, "model_name": "claude-opus-5", "output_tokens": 236, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:32.626434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02057, "created_at": "2026-09-17T03:11:32.626Z" }, { "sequence": 587, "attempt_id": "d439d5e9-f13b-4e98-9760-3167f611626a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00705825, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 6.068606765940785, "model_name": "muse-spark-1.2", "output_tokens": 909, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:33.327114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00705825, "created_at": "2026-09-17T03:11:33.327Z" }, { "sequence": 588, "attempt_id": "513b86d0-688b-4fd5-b253-002ea7fe6697", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7882176489802077, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:11:33.461956+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003789, "created_at": "2026-09-17T03:11:33.462Z" }, { "sequence": 589, "attempt_id": "5e077bea-4e73-413e-a5bb-f0d70aa86aab", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006408000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.0603284869575873, "model_name": "gpt-5.6-luna", "output_tokens": 123, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:33.512133+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006408000000000001, "created_at": "2026-09-17T03:11:33.512Z" }, { "sequence": 590, "attempt_id": "dba0e142-c7f1-4615-babc-926a3cc77074", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010144, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.746087564039044, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:33.683072+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010144, "created_at": "2026-09-17T03:11:33.683Z" }, { "sequence": 591, "attempt_id": "f842c1b3-8c54-4f03-9af7-dbcf8f1b17c2", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00208725, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.5887570660561323, "model_name": "gemini-3.8-flash", "output_tokens": 314, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:11:33.732115+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00208725, "created_at": "2026-09-17T03:11:33.732Z" }, { "sequence": 592, "attempt_id": "a146277c-41d9-40c2-8de1-1185b7a238f6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005388, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 5.489650465082377, "model_name": "gpt-5.6-terra", "output_tokens": 38, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:35.638716+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005388, "created_at": "2026-09-17T03:11:35.638Z" }, { "sequence": 593, "attempt_id": "4e93f6a0-3743-4dc7-b567-b72716888126", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0136205, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 29.440734864911065, "model_name": "muse-spark-1.3", "output_tokens": 2461, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:36.073376+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0136205, "created_at": "2026-09-17T03:11:36.073Z" }, { "sequence": 594, "attempt_id": "b11edaca-4777-4057-9a54-f6af3019c845", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017455, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.8455820289673284, "model_name": "claude-opus-5", "output_tokens": 133, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:36.578133+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017455, "created_at": "2026-09-17T03:11:36.578Z" }, { "sequence": 595, "attempt_id": "2e1fb529-9556-473c-92b4-dc91a2fc141f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006952, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.81479960703291, "model_name": "claude-sonnet-5", "output_tokens": 130, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:36.629555+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006952, "created_at": "2026-09-17T03:11:36.633Z" }, { "sequence": 596, "attempt_id": "5a679139-aff9-418c-9cc5-38621fa516f6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 1.1949724860023707, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:36.884315+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001856, "created_at": "2026-09-17T03:11:36.884Z" }, { "sequence": 597, "attempt_id": "7d5fd89d-1dbe-4acb-acbc-80bb67ae2ca1", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008798, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.056827415013686, "model_name": "gemini-3.1-pro-preview", "output_tokens": 531, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:11:37.523267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008798, "created_at": "2026-09-17T03:11:37.523Z" }, { "sequence": 598, "attempt_id": "bb7f962a-8d8f-4457-8a7b-52a82cdc9be5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004936000000000001, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.2781754810130224, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:38.203191+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004936000000000001, "created_at": "2026-09-17T03:11:38.203Z" }, { "sequence": 599, "attempt_id": "7a95dcdd-a6be-4c32-8722-394e4fd070de", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005428, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.5712887500412762, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:38.253135+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005428, "created_at": "2026-09-17T03:11:38.254Z" }, { "sequence": 600, "attempt_id": "1e014794-c660-4ae7-875a-9b62c3db59f5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00665875, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 11.052655894076452, "model_name": "muse-spark-1.3", "output_tokens": 815, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:38.299228+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00665875, "created_at": "2026-09-17T03:11:38.299Z" }, { "sequence": 601, "attempt_id": "24745a6b-7d5a-4663-8298-6b742ee25d48", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010916, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.2026300800498575, "model_name": "gpt-5.6-sol", "output_tokens": 69, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:38.832613+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010916, "created_at": "2026-09-17T03:11:38.832Z" }, { "sequence": 602, "attempt_id": "57243f8e-8cbf-4613-a2e6-b7bb3c50575d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.7960829819785431, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:39.141915+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003668, "created_at": "2026-09-17T03:11:39.142Z" }, { "sequence": 603, "attempt_id": "69691359-7df5-418e-ac93-d6fccac4694e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02973, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.628084594965912, "model_name": "claude-fable-5-1", "output_tokens": 29, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:39.193995+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02973, "created_at": "2026-09-17T03:11:39.194Z" }, { "sequence": 604, "attempt_id": "ff451f7c-1f8f-4240-8004-62917703c933", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02669, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.1573236900148913, "model_name": "gpt-6-astra", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:39.278572+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02669, "created_at": "2026-09-17T03:11:39.278Z" }, { "sequence": 605, "attempt_id": "63bf5cf9-7d42-4501-a019-a4624552174f", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.8663034898927435, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:41.108365+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014315, "created_at": "2026-09-17T03:11:41.108Z" }, { "sequence": 606, "attempt_id": "71ffbf0f-2b3e-4f1a-90a4-8c254f2ac8df", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.9099239059723914, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:41.236945+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T03:11:41.237Z" }, { "sequence": 607, "attempt_id": "1cf9f7a1-1ceb-4886-a912-75ab5af658bb", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.708524595014751, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:41.859142+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T03:11:41.859Z" }, { "sequence": 608, "attempt_id": "755af591-aad7-41fe-8f18-1cca2330f2e1", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.4513665140839294, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:42.645743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T03:11:42.645Z" }, { "sequence": 609, "attempt_id": "de2d5280-ce0e-4765-a57d-92026d4f03b5", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6713304129661992, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:42.950914+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:11:42.951Z" }, { "sequence": 610, "attempt_id": "81724fef-ed20-41e3-821a-5dbb1287debd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3342536769341677, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:43.241873+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:11:43.242Z" }, { "sequence": 611, "attempt_id": "fa918a4c-4bf2-4deb-9bf9-50efbc754a69", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018718, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 15.27232546696905, "model_name": "claude-sonnet-5", "output_tokens": 1285, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:43.577360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.018718, "created_at": "2026-09-17T03:11:43.577Z" }, { "sequence": 612, "attempt_id": "98661232-46d8-4ed8-b030-092545bfc8a1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01024575, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.075815274030901, "model_name": "muse-spark-1.2", "output_tokens": 1644, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:11:43.627565+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01024575, "created_at": "2026-09-17T03:11:43.627Z" }, { "sequence": 613, "attempt_id": "f74bd936-711c-4b31-a1df-11fc1473f177", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00591075, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 5.384695586981252, "model_name": "gemini-3.8-flash", "output_tokens": 1350, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:43.679571+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00591075, "created_at": "2026-09-17T03:11:43.679Z" }, { "sequence": 614, "attempt_id": "c36a8e20-06bb-4e83-9fad-67bceb69337f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.8643219399964437, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:43.860369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:11:43.860Z" }, { "sequence": 615, "attempt_id": "c28d5cf5-dd5b-4653-881b-99e9f2991acf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0052, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2198352189734578, "model_name": "gpt-5.6-terra", "output_tokens": 35, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:43.916966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0052, "created_at": "2026-09-17T03:11:43.917Z" }, { "sequence": 616, "attempt_id": "b510300e-1679-4bc9-91a8-0b92b43e1a81", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.847441611927934, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:11:44.527403+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:11:44.527Z" }, { "sequence": 617, "attempt_id": "0ac9c1ec-bf80-42bd-8029-4d542f2a8b3a", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00197025, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.103905054042116, "model_name": "gemini-3.8-flash", "output_tokens": 298, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:45.731822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00197025, "created_at": "2026-09-17T03:11:45.731Z" }, { "sequence": 618, "attempt_id": "0af2bec9-ce7a-4953-aa93-4630f76368a0", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00470225, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.733842964982614, "model_name": "muse-spark-1.2", "output_tokens": 362, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:46.645861+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00470225, "created_at": "2026-09-17T03:11:46.646Z" }, { "sequence": 619, "attempt_id": "d51a6d60-3275-4813-9e77-6e7f96a39a28", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-013", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.729100083000958, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:47.425355+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T03:11:47.425Z" }, { "sequence": 620, "attempt_id": "d563dcd0-82dc-46e5-815b-547b693543a7", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.0001559000229463, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:47.775951+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00601, "created_at": "2026-09-17T03:11:47.776Z" }, { "sequence": 621, "attempt_id": "3f13bf44-7e89-4858-b533-f9b64641138b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016175, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.7886356649687514, "model_name": "claude-opus-5", "output_tokens": 57, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:48.378055+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016175, "created_at": "2026-09-17T03:11:48.378Z" }, { "sequence": 622, "attempt_id": "219c776b-22b5-40dd-8521-444ae766670e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.511069120024331, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:48.475940+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:11:48.476Z" }, { "sequence": 623, "attempt_id": "f6564566-fa89-4791-ae41-468a20acd004", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00505075, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 5.097933196928352, "model_name": "muse-spark-1.3", "output_tokens": 444, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:48.825412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00505075, "created_at": "2026-09-17T03:11:48.825Z" }, { "sequence": 624, "attempt_id": "fdd6a016-b6b4-46d9-9f7e-79f011cb43d2", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006174, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 6.0739705869928, "model_name": "gemini-3.1-pro-preview", "output_tokens": 325, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:49.373121+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006174, "created_at": "2026-09-17T03:11:49.373Z" }, { "sequence": 625, "attempt_id": "ad2555bb-07b9-420a-8faf-9ba1039b8714", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.6397106590447947, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:50.114809+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:11:50.115Z" }, { "sequence": 626, "attempt_id": "48ab25e0-a773-4743-b296-a7446b36454a", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.9013200149638578, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:11:51.067345+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:11:51.067Z" }, { "sequence": 627, "attempt_id": "a94344aa-385c-4204-b505-13a6a5c1fb75", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00175875, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.7359063269104809, "model_name": "gemini-3.8-flash", "output_tokens": 227, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:51.160435+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00175875, "created_at": "2026-09-17T03:11:51.160Z" }, { "sequence": 628, "attempt_id": "2f5e3f42-7dd7-464b-af73-d88c9e81f8b2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 3.583853736985475, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:51.417430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:11:51.417Z" }, { "sequence": 629, "attempt_id": "8d1bc562-a94a-40b1-9d8d-2d6baa69e1b1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005754, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.942950607975945, "model_name": "gpt-5.6-luna", "output_tokens": 69, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:51.469272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005754, "created_at": "2026-09-17T03:11:51.469Z" }, { "sequence": 630, "attempt_id": "d1d8a6bc-5cf8-4e2a-9bf0-b05ee521c204", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005802, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 3.7327629489591345, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:52.161335+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005802, "created_at": "2026-09-17T03:11:52.161Z" }, { "sequence": 631, "attempt_id": "7eace9c9-b330-4ca6-9d78-3c71575e4b0f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005756, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.175443466985598, "model_name": "gemini-3.1-pro-preview", "output_tokens": 278, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:53.053637+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005756, "created_at": "2026-09-17T03:11:53.053Z" }, { "sequence": 632, "attempt_id": "305dd185-67fa-46d8-a19a-e5cee5c5828a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.6990754149155691, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:53.803427+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T03:11:53.803Z" }, { "sequence": 633, "attempt_id": "5ff78317-6326-45fd-b28f-50aa4a7873c4", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0116355, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 20.44383939204272, "model_name": "muse-spark-1.3", "output_tokens": 1971, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:53.857315+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0116355, "created_at": "2026-09-17T03:11:53.857Z" }, { "sequence": 634, "attempt_id": "7a992ee6-beb0-42eb-b2d1-65e816cff801", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015995, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.937611851026304, "model_name": "claude-opus-5", "output_tokens": 77, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:54.445851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015995, "created_at": "2026-09-17T03:11:54.446Z" }, { "sequence": 635, "attempt_id": "97714ec5-d959-4b33-a918-d16ad2a2ca4d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005258, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 3.327465069014579, "model_name": "muse-spark-1.2", "output_tokens": 471, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:54.528527+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005258, "created_at": "2026-09-17T03:11:54.528Z" }, { "sequence": 636, "attempt_id": "2ac01a15-87ef-409b-bc33-128c14037420", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006408, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.375635551987216, "model_name": "claude-sonnet-5", "output_tokens": 78, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:54.580323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006408, "created_at": "2026-09-17T03:11:54.580Z" }, { "sequence": 637, "attempt_id": "7b18e6ff-6766-4d02-9788-aafe0bfe20d9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.8889983179979026, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:55.469646+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:11:55.469Z" }, { "sequence": 638, "attempt_id": "9fd9fba2-0d81-4556-920d-ff831904a376", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005294, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.480797695927322, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:55.980374+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005294, "created_at": "2026-09-17T03:11:55.980Z" }, { "sequence": 639, "attempt_id": "d32100ec-3fc1-4f23-8465-3d019136a0d8", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.2669507779646665, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:56.124629+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:11:56.124Z" }, { "sequence": 640, "attempt_id": "8fc764dc-fd48-4390-9405-bbd0b35acde0", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00569575, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 5.317758035031147, "model_name": "muse-spark-1.3", "output_tokens": 574, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:56.437033+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00569575, "created_at": "2026-09-17T03:11:56.437Z" }, { "sequence": 641, "attempt_id": "11beff31-6099-4124-a95a-80c932db4c91", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 5.085992314037867, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:56.545569+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T03:11:56.545Z" }, { "sequence": 642, "attempt_id": "24d6328c-0db5-426b-b9ff-19d09faac6f9", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001572, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.464345145970583, "model_name": "gemini-3.8-flash", "output_tokens": 192, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:56.977525+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001572, "created_at": "2026-09-17T03:11:56.977Z" }, { "sequence": 643, "attempt_id": "04a2f249-05e2-498b-a048-034ce8cfc7b4", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.0627019400708377, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:11:57.084078+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003683, "created_at": "2026-09-17T03:11:57.084Z" }, { "sequence": 644, "attempt_id": "1c1f03e2-3ce9-4ddf-a93d-e420e9dce4aa", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 3.225650139967911, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:57.134739+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:11:57.134Z" }, { "sequence": 645, "attempt_id": "a3c4d2aa-b168-44a7-8d0e-96030583cca1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.7445697099901736, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:57.918891+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001884, "created_at": "2026-09-17T03:11:57.919Z" }, { "sequence": 646, "attempt_id": "29fdc131-9c67-4bee-953e-0971a3fcdceb", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006412, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.614468805957586, "model_name": "gemini-3.1-pro-preview", "output_tokens": 345, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "google", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:59.236085+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006412, "created_at": "2026-09-17T03:11:59.236Z" }, { "sequence": 647, "attempt_id": "241c4d77-861b-4e55-afeb-c7e90243b0f8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049305, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 2.9195223449496552, "model_name": "muse-spark-1.2", "output_tokens": 416, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:59.397515+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049305, "created_at": "2026-09-17T03:11:59.397Z" }, { "sequence": 648, "attempt_id": "3b96004e-e1a0-4bf2-b4e3-04ebe03894ee", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.007026, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 2.754816284053959, "model_name": "claude-sonnet-5", "output_tokens": 129, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:59.880106+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007026, "created_at": "2026-09-17T03:11:59.880Z" }, { "sequence": 649, "attempt_id": "6936121a-ab60-42a8-8b92-8e832ed0ccae", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01739, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.323509279987775, "model_name": "claude-opus-5", "output_tokens": 122, "prediction": { "choice": "C" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:00.345513+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01739, "created_at": "2026-09-17T03:12:00.345Z" }, { "sequence": 650, "attempt_id": "d64adb11-9b1d-4d9b-849e-47700644dcc2", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02634, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.434617858962156, "model_name": "gpt-6-astra", "output_tokens": 44, "prediction": { "choice": "C" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:00.396879+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02634, "created_at": "2026-09-17T03:12:00.397Z" }, { "sequence": 651, "attempt_id": "54d8d572-d7c3-4c19-8986-c43680057cac", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01977025, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 22.096016245079227, "model_name": "muse-spark-1.2", "output_tokens": 3908, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:00.976733+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01977025, "created_at": "2026-09-17T03:12:00.976Z" }, { "sequence": 652, "attempt_id": "9416e1fb-e6e2-4b6e-b94d-8268cb8254f3", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0296, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 4.622650080942549, "model_name": "claude-fable-5-1", "output_tokens": 18, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:01.209110+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0296, "created_at": "2026-09-17T03:12:01.209Z" }, { "sequence": 653, "attempt_id": "03aae3c5-7bd6-4d82-b2fc-70482b4c3879", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005908, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.8431205149972811, "model_name": "gpt-5.6-terra", "output_tokens": 90, "prediction": { "choice": "A" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:01.285954+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005908, "created_at": "2026-09-17T03:12:01.286Z" }, { "sequence": 654, "attempt_id": "d109a398-09d8-48f3-921e-ea212629b67c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004701, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 5.152085245004855, "model_name": "muse-spark-1.3", "output_tokens": 362, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:01.326369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004701, "created_at": "2026-09-17T03:12:01.326Z" }, { "sequence": 655, "attempt_id": "1705a497-8ab5-41a0-9bf1-edab845af9cd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010996, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.100116626941599, "model_name": "gpt-5.6-sol", "output_tokens": 67, "prediction": { "choice": "C" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:01.384787+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010996, "created_at": "2026-09-17T03:12:01.384Z" }, { "sequence": 656, "attempt_id": "0b4146f8-ed18-49fc-955a-8813722e3544", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.7934254489373416, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:01.825206+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003761, "created_at": "2026-09-17T03:12:01.825Z" }, { "sequence": 657, "attempt_id": "c77ac3a0-3c27-4388-8eec-c8199dda1b5b", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.033642, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 24.531029382022098, "model_name": "gemini-3.1-pro-preview", "output_tokens": 2615, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:02.103858+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.033642, "created_at": "2026-09-17T03:12:02.104Z" }, { "sequence": 658, "attempt_id": "6996de0c-e567-4109-a3e0-9396be04cc25", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.6955347320763394, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:02.843509+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T03:12:02.843Z" }, { "sequence": 659, "attempt_id": "230d9e5e-5472-415f-81d7-da5f247e5717", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.5076094190590084, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:03.931681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01434, "created_at": "2026-09-17T03:12:03.931Z" }, { "sequence": 660, "attempt_id": "68e9327c-965e-4e2c-8c23-a3fc1ccbd6fd", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.1839894120348617, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:04.055103+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005736, "created_at": "2026-09-17T03:12:04.055Z" }, { "sequence": 661, "attempt_id": "ce47a393-59ae-4d70-967b-9f2b6b717d05", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0044565, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 3.9627354600233957, "model_name": "gemini-3.8-flash", "output_tokens": 956, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:04.410283+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0044565, "created_at": "2026-09-17T03:12:04.410Z" }, { "sequence": 662, "attempt_id": "5ad2b0a7-1ba5-404e-9341-8bfd676ffbf7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005632, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 4.537616326007992, "model_name": "gpt-5.6-luna", "output_tokens": 67, "prediction": { "choice": "C" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:04.479513+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005632, "created_at": "2026-09-17T03:12:04.479Z" }, { "sequence": 663, "attempt_id": "a02d0941-9ab8-4f62-8a4e-f0f8a85de5cd", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.6109942119801417, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:04.526082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:12:04.526Z" }, { "sequence": 664, "attempt_id": "21aaaeaa-1c30-4013-8bc5-e04970b86109", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.4309307100484148, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:04.797927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T03:12:04.798Z" }, { "sequence": 665, "attempt_id": "40250bf5-2ac1-4c5e-992e-966f443b027e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 0.9240714509505779, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:05.376771+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:12:05.376Z" }, { "sequence": 666, "attempt_id": "ed223da5-8b29-4f88-a2ac-1eb0960de1d7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.8293457370018587, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:05.804704+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:12:05.804Z" }, { "sequence": 667, "attempt_id": "81226e7b-a3fa-4245-ac0a-515987d97be4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005458, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.7601991120027378, "model_name": "gpt-5.6-terra", "output_tokens": 56, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:05.856991+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005458, "created_at": "2026-09-17T03:12:05.857Z" }, { "sequence": 668, "attempt_id": "ce1db3b1-7964-4e39-93c5-bbab06207250", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.049276147969067, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:05.897545+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:12:05.897Z" }, { "sequence": 669, "attempt_id": "d11117b7-b091-43bf-a7e6-9ebfaf2a5b6e", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.009032, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 6.518618904054165, "model_name": "gemini-3.1-pro-preview", "output_tokens": 559, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:06.915897+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009032, "created_at": "2026-09-17T03:12:06.916Z" }, { "sequence": 670, "attempt_id": "3ba6046a-0a09-4fa6-bab0-3e20aa3fec60", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044055, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 1.7928618530277163, "model_name": "muse-spark-1.2", "output_tokens": 291, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:07.641289+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0044055, "created_at": "2026-09-17T03:12:07.641Z" }, { "sequence": 671, "attempt_id": "64cb751c-8100-4094-84aa-80d5a71d180e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.924071489018388, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:08.884190+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T03:12:08.884Z" }, { "sequence": 672, "attempt_id": "c3aa0855-34e1-4894-89e0-f7fe4889710e", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 1.2270294670015574, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:08.928489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T03:12:08.928Z" }, { "sequence": 673, "attempt_id": "867d14a5-37c6-4a17-9297-082f3fd627ff", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004873, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.6025799569906667, "model_name": "muse-spark-1.3", "output_tokens": 401, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:09.023124+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004873, "created_at": "2026-09-17T03:12:09.023Z" }, { "sequence": 674, "attempt_id": "7f649d36-7fbc-4773-8611-22c80071f864", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006804, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.707865001983009, "model_name": "gemini-3.1-pro-preview", "output_tokens": 377, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:09.234387+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006804, "created_at": "2026-09-17T03:12:09.234Z" }, { "sequence": 675, "attempt_id": "ee4cb311-7b32-4043-a4ea-d8ca7be8baef", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0096465, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 8.441316016949713, "model_name": "muse-spark-1.2", "output_tokens": 1518, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:09.767986+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0096465, "created_at": "2026-09-17T03:12:09.768Z" }, { "sequence": 676, "attempt_id": "6241930a-25d5-466d-b98d-6eef3bada933", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017515, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 4.401619579992257, "model_name": "claude-opus-5", "output_tokens": 139, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:10.341857+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017515, "created_at": "2026-09-17T03:12:10.342Z" }, { "sequence": 677, "attempt_id": "8db1f38b-4029-4d3b-9519-c72bbbaef44b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02875, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 4.505828456021845, "model_name": "claude-fable-5-1", "output_tokens": 13, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:10.403674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02875, "created_at": "2026-09-17T03:12:10.403Z" }, { "sequence": 678, "attempt_id": "17130f49-fddc-4413-8326-631f078d50d3", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00147, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 6.107604110962711, "model_name": "gemini-3.8-flash", "output_tokens": 164, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:10.676944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00147, "created_at": "2026-09-17T03:12:10.677Z" }, { "sequence": 679, "attempt_id": "d4eea988-4fb8-47dd-9d49-802299dcf873", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005428, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7608591540483758, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:10.825085+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005428, "created_at": "2026-09-17T03:12:10.825Z" }, { "sequence": 680, "attempt_id": "2b55b9bb-dd9e-404b-92e7-66c7f4f1b4e8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.9362348510185257, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:11.213600+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:12:11.213Z" }, { "sequence": 681, "attempt_id": "574f98cf-fa37-4680-8d0a-80b8ad439784", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02635, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.4099687569541857, "model_name": "gpt-6-astra", "output_tokens": 49, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:11.338776+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02635, "created_at": "2026-09-17T03:12:11.338Z" }, { "sequence": 682, "attempt_id": "b5ad4f68-1111-4fcb-90bd-d5a8f061bfc1", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.006069257040508, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:11.451719+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:12:11.451Z" }, { "sequence": 683, "attempt_id": "3b4b63ec-1de7-4bae-bc8b-c0194d2e9a6a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 3.0598668809980154, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:12.030020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:12:12.030Z" }, { "sequence": 684, "attempt_id": "1e061d98-c826-4d1e-8328-671803093fc3", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7548836969071999, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:12.828570+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:12:12.828Z" }, { "sequence": 685, "attempt_id": "d45e4555-13f6-498c-906e-c66e0331904e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0153375, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 35.40869758196641, "model_name": "muse-spark-1.3", "output_tokens": 2865, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:13.756755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0153375, "created_at": "2026-09-17T03:12:13.756Z" }, { "sequence": 686, "attempt_id": "45d0432e-2af1-4695-9536-c807521bbdd1", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006858, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.502908758004196, "model_name": "claude-sonnet-5", "output_tokens": 99, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:13.995844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006858, "created_at": "2026-09-17T03:12:13.996Z" }, { "sequence": 687, "attempt_id": "9b93a273-9e37-40a4-be3f-6a1f94ccb895", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7539259750628844, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:14.626025+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:12:14.626Z" }, { "sequence": 688, "attempt_id": "b037231d-a2ab-47b5-b911-32dee82103cd", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003384, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.446592629072256, "model_name": "gemini-3.8-flash", "output_tokens": 675, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:14.830730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003384, "created_at": "2026-09-17T03:12:14.830Z" }, { "sequence": 689, "attempt_id": "e2503402-9df7-4896-83ff-c14b880779bf", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.0932785290060565, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:12:15.349526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:12:15.349Z" }, { "sequence": 690, "attempt_id": "2768cacc-28b3-42fa-900e-74ceedda0520", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00771, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.687882098020054, "model_name": "gemini-3.1-pro-preview", "output_tokens": 453, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:15.498366+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00771, "created_at": "2026-09-17T03:12:15.498Z" }, { "sequence": 691, "attempt_id": "de2c4e0c-a160-48bc-a145-bb294143c2d5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00564, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.820615406963043, "model_name": "gpt-5.6-terra", "output_tokens": 59, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:15.859431+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00564, "created_at": "2026-09-17T03:12:15.859Z" }, { "sequence": 692, "attempt_id": "26c50489-63fd-4b65-9a33-4c02d74b0ee2", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8416627470869571, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:12:16.384099+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:12:16.384Z" }, { "sequence": 693, "attempt_id": "4f60152b-3e20-43d5-baf9-e8fb616b9af7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011244, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.586192777962424, "model_name": "gpt-5.6-sol", "output_tokens": 69, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:16.427847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011244, "created_at": "2026-09-17T03:12:16.428Z" }, { "sequence": 694, "attempt_id": "fc95d136-b5d8-46ba-a612-3ba4271cecf3", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001656, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.5664854989154264, "model_name": "gemini-3.8-flash", "output_tokens": 199, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:12:16.959825+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001656, "created_at": "2026-09-17T03:12:16.959Z" }, { "sequence": 695, "attempt_id": "85f964a8-f9e6-4315-9a0b-dee6cbe0f122", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016395, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 6.198671592050232, "model_name": "claude-opus-5", "output_tokens": 69, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:17.581879+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016395, "created_at": "2026-09-17T03:12:17.582Z" }, { "sequence": 696, "attempt_id": "de9f7a45-b4af-44de-b1f6-31a13690d598", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005892, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.194904412026517, "model_name": "gpt-5.6-luna", "output_tokens": 80, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:17.863539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005892, "created_at": "2026-09-17T03:12:17.863Z" }, { "sequence": 697, "attempt_id": "47d07457-fd39-46ff-bbe9-bf413dea8429", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9026185399852693, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:18.906314+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:12:18.906Z" }, { "sequence": 698, "attempt_id": "aad6ceef-b42b-46c5-9cd9-2f355956bafa", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0083445, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 8.266924450988881, "model_name": "muse-spark-1.2", "output_tokens": 1219, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:19.134058+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0083445, "created_at": "2026-09-17T03:12:19.134Z" }, { "sequence": 699, "attempt_id": "b2f8d4f0-bc84-4dc6-bdc0-4f1511f4c20b", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.6545872570713982, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:19.279698+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:12:19.279Z" }, { "sequence": 700, "attempt_id": "b918fa0d-0989-4d8b-b637-d5e9e5692542", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006878, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 4.653898206073791, "model_name": "gemini-3.1-pro-preview", "output_tokens": 371, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:12:19.531248+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006878, "created_at": "2026-09-17T03:12:19.531Z" }, { "sequence": 701, "attempt_id": "28f24581-3182-43e7-92ee-bc7a1a8828ab", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.6774215989280492, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:19.586642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:12:19.588Z" }, { "sequence": 702, "attempt_id": "542693e9-8671-442c-8d4b-7c3df359db07", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.175597527064383, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:19.645218+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:12:19.645Z" }, { "sequence": 703, "attempt_id": "ec285d4b-d44f-4d15-9052-768d2e9bf348", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.3061421690508723, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:20.640495+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:12:20.640Z" }, { "sequence": 704, "attempt_id": "7e60ab6a-1c20-4cf4-954b-747a4c333553", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.488964777905494, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:20.695394+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:12:20.695Z" }, { "sequence": 705, "attempt_id": "20119b59-b3f2-40d9-b92c-f63a2837b597", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00676775, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 10.128204443957657, "model_name": "muse-spark-1.3", "output_tokens": 848, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:20.850788+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00676775, "created_at": "2026-09-17T03:12:20.851Z" }, { "sequence": 706, "attempt_id": "6d4fbce1-50b4-4f63-80e6-be767679d1ff", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0056855, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 5.02281672495883, "model_name": "muse-spark-1.3", "output_tokens": 571, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:20.933900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0056855, "created_at": "2026-09-17T03:12:20.934Z" }, { "sequence": 707, "attempt_id": "15d29632-a5e5-4973-b105-f6432bdbfd9b", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00117675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.3043932190630585, "model_name": "gemini-3.8-flash", "output_tokens": 87, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:21.002000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00117675, "created_at": "2026-09-17T03:12:21.002Z" }, { "sequence": 708, "attempt_id": "321734dc-bc41-44ab-bcc4-8b59b96572a7", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.3289480740204453, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:21.290813+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:12:21.291Z" }, { "sequence": 709, "attempt_id": "d0298a5a-0329-4074-88d3-e7a930220a89", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00622525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 4.924019098980352, "model_name": "muse-spark-1.2", "output_tokens": 698, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:12:21.352185+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00622525, "created_at": "2026-09-17T03:12:21.352Z" }, { "sequence": 710, "attempt_id": "e712d821-a574-4c4b-a2b4-3127b688f540", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.4561704529915005, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:22.043272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:12:22.043Z" }, { "sequence": 711, "attempt_id": "93c7d26d-369d-44d4-ad9e-3bec6376aa87", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.4193870819872245, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:22.115103+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:12:22.115Z" }, { "sequence": 712, "attempt_id": "b33d0a67-53bf-4e2e-a4c8-8c67a8a03df1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.578466056031175, "model_name": "muse-spark-1.2", "output_tokens": 180, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:22.483618+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003925, "created_at": "2026-09-17T03:12:22.483Z" }, { "sequence": 713, "attempt_id": "a5febf66-1d93-44de-a9eb-bd6beee97fd1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.4818869809387252, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:22.902084+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:12:22.902Z" }, { "sequence": 714, "attempt_id": "85f2f064-ecd0-4283-aaab-4e0cbd232be0", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003396, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.4112681599799544, "model_name": "gemini-3.1-pro-preview", "output_tokens": 94, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:23.052318+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003396, "created_at": "2026-09-17T03:12:23.052Z" }, { "sequence": 715, "attempt_id": "90c71a40-bd48-4c97-a1d8-aab447337775", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006128, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 1.907714041066356, "model_name": "claude-sonnet-5", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:23.254031+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006128, "created_at": "2026-09-17T03:12:23.254Z" }, { "sequence": 716, "attempt_id": "e7a80fae-8070-4718-912a-b05983348cef", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.794700940954499, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:23.753442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004968, "created_at": "2026-09-17T03:12:23.753Z" }, { "sequence": 717, "attempt_id": "a71d28ab-2b2d-440e-85c9-5a56e75189a0", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.8543293359689415, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:23.909892+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01447, "created_at": "2026-09-17T03:12:23.910Z" }, { "sequence": 718, "attempt_id": "c7c4540e-c021-485c-a252-145d56a2abd4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.4238748570205644, "model_name": "gpt-5.6-terra", "output_tokens": 53, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:23.966876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005436, "created_at": "2026-09-17T03:12:23.967Z" }, { "sequence": 719, "attempt_id": "27ec30d9-2569-4644-b0ac-af991ce9adcb", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.53178681305144, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:24.517734+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:12:24.517Z" }, { "sequence": 720, "attempt_id": "ae61d69c-8543-42c0-9e9c-caef1c7705ac", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7340271410066634, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:24.584450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:12:24.584Z" }, { "sequence": 721, "attempt_id": "26c1d202-2d56-48a1-a252-3a0cfd21ac73", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.764280627015978, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:24.934075+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:12:24.934Z" }, { "sequence": 722, "attempt_id": "909ecd82-23b9-4d19-b5c9-f98bb6d8e51b", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0015975, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.7683970330981538, "model_name": "gemini-3.8-flash", "output_tokens": 196, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:25.095526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0015975, "created_at": "2026-09-17T03:12:25.095Z" }, { "sequence": 723, "attempt_id": "f5315663-fc8d-4392-ba1f-1a278f2e5768", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.407146377954632, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:25.507730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:12:25.507Z" }, { "sequence": 724, "attempt_id": "cde55760-168e-4b21-94ba-0f1a976b887f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.6840691700344905, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:25.836048+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001856, "created_at": "2026-09-17T03:12:25.836Z" }, { "sequence": 725, "attempt_id": "8f88f4f3-513e-4d7d-bb7a-62f11859ac77", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005576, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.7415951860602945, "model_name": "gemini-3.1-pro-preview", "output_tokens": 273, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:26.850268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005576, "created_at": "2026-09-17T03:12:26.850Z" }, { "sequence": 726, "attempt_id": "db30dfd9-0dab-42a1-b7d6-87d86de6dec8", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016005, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.8726579380454496, "model_name": "claude-opus-5", "output_tokens": 75, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:27.511814+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016005, "created_at": "2026-09-17T03:12:27.512Z" }, { "sequence": 727, "attempt_id": "c3bf2338-1b9b-41f9-9610-604afef74308", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006362, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.7251612109830603, "model_name": "claude-sonnet-5", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:27.714914+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006362, "created_at": "2026-09-17T03:12:27.715Z" }, { "sequence": 728, "attempt_id": "b8b00dcf-92de-4e59-b18e-3ab747fbd9d5", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.3566450669895858, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:27.923126+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02454, "created_at": "2026-09-17T03:12:27.923Z" }, { "sequence": 729, "attempt_id": "e76b43af-ec76-445d-b45d-e02c0043fe70", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005464, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.6048626960255206, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:28.514609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005464, "created_at": "2026-09-17T03:12:28.514Z" }, { "sequence": 730, "attempt_id": "222ec057-12aa-4d18-ab08-96d95ef272e8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010896, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.751320852083154, "model_name": "gpt-5.6-sol", "output_tokens": 68, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:28.643715+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010896, "created_at": "2026-09-17T03:12:28.643Z" }, { "sequence": 731, "attempt_id": "74a3225c-f6e6-4daf-9c3a-948956ef79bd", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 1.028691554092802, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:12:29.599385+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003668, "created_at": "2026-09-17T03:12:29.599Z" }, { "sequence": 732, "attempt_id": "577a75e5-8798-430a-ad43-8e3e0f1f9f82", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02963, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.540425140992738, "model_name": "claude-fable-5-1", "output_tokens": 27, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:30.125283+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02963, "created_at": "2026-09-17T03:12:30.125Z" }, { "sequence": 733, "attempt_id": "f0bf37c8-aca8-4f71-ab5d-598226240a6f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0039675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 9.459931844030507, "model_name": "muse-spark-1.3", "output_tokens": 190, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:30.209062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0039675, "created_at": "2026-09-17T03:12:30.209Z" }, { "sequence": 734, "attempt_id": "8b541b41-ffd2-4613-91ff-2d28b7beed1e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00808075, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.784529292024672, "model_name": "muse-spark-1.2", "output_tokens": 1154, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:30.805434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00808075, "created_at": "2026-09-17T03:12:30.805Z" }, { "sequence": 735, "attempt_id": "8ca48553-681d-45e0-98a6-0cf4dce26347", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.375773865962401, "model_name": "gpt-5.6-luna", "output_tokens": 75, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:30.944526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005668, "created_at": "2026-09-17T03:12:30.944Z" }, { "sequence": 736, "attempt_id": "f4653d73-7cc0-43e3-ab2f-08b91bdb982d", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 1.224790649022907, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:32.243852+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001876, "created_at": "2026-09-17T03:12:32.244Z" }, { "sequence": 737, "attempt_id": "a2f9cbd8-dd03-44c0-baa2-3cce09650e63", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01557, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.70322943397332, "model_name": "claude-opus-5", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:32.967586+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01557, "created_at": "2026-09-17T03:12:32.967Z" }, { "sequence": 738, "attempt_id": "f0fda2ab-544f-4523-be17-83e7c33a8377", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00409575, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 6.167217475012876, "model_name": "gemini-3.8-flash", "output_tokens": 866, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:12:34.146340+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00409575, "created_at": "2026-09-17T03:12:34.146Z" }, { "sequence": 739, "attempt_id": "6b65e6f7-91d1-435c-a156-c24d88226f25", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008068, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.3190621440298855, "model_name": "claude-sonnet-5", "output_tokens": 238, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:34.204824+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008068, "created_at": "2026-09-17T03:12:34.204Z" }, { "sequence": 740, "attempt_id": "e6ed0a65-294e-45dd-b414-3e2ee8af52e2", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02639, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.207453773007728, "model_name": "gpt-6-astra", "output_tokens": 47, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:34.510614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02639, "created_at": "2026-09-17T03:12:34.510Z" }, { "sequence": 741, "attempt_id": "b5559680-7ddb-449a-a76a-983a962bd261", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 4.640244425972924, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:34.822499+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02901, "created_at": "2026-09-17T03:12:34.822Z" }, { "sequence": 742, "attempt_id": "5eb37343-0d25-4d44-a3e3-3f83f2ecc4b4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.37305437005125, "model_name": "gpt-5.6-sol", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:35.399617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011056, "created_at": "2026-09-17T03:12:35.399Z" }, { "sequence": 743, "attempt_id": "527c4f11-fc60-4fa9-9ca9-91f2ac5fa1eb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005696000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.5183978480054066, "model_name": "gpt-5.6-luna", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:35.785213+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005696000000000001, "created_at": "2026-09-17T03:12:35.785Z" }, { "sequence": 744, "attempt_id": "3aad628c-9bfa-4042-9e42-186cb569ef36", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005624, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.9622918310342357, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:36.167492+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005624, "created_at": "2026-09-17T03:12:36.167Z" }, { "sequence": 745, "attempt_id": "938f4320-fe5e-4bc9-b1a0-d7591a335db9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003615, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.8909002239815891, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:36.352317+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003615, "created_at": "2026-09-17T03:12:36.352Z" }, { "sequence": 746, "attempt_id": "6272790c-6110-4c3e-9bf9-79abebb50a88", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0085545, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 6.75144525396172, "model_name": "muse-spark-1.2", "output_tokens": 1269, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:36.410561+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0085545, "created_at": "2026-09-17T03:12:36.410Z" }, { "sequence": 747, "attempt_id": "ae0fc597-1cca-4820-b01b-a9fa97572dab", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00306375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 2.3387164120795205, "model_name": "gemini-3.8-flash", "output_tokens": 586, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:12:37.241431+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00306375, "created_at": "2026-09-17T03:12:37.241Z" }, { "sequence": 748, "attempt_id": "13d02ded-821d-4b2c-aff7-83d4270a617c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00698425, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 13.6606033280259, "model_name": "muse-spark-1.3", "output_tokens": 896, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:37.627833+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00698425, "created_at": "2026-09-17T03:12:37.628Z" }, { "sequence": 749, "attempt_id": "a3e59f62-da96-48b1-8a1f-e5dbc8acdfe2", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.6509498879313469, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:38.333998+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T03:12:38.334Z" }, { "sequence": 750, "attempt_id": "aeb57fe7-5377-46c0-86da-078428aa11b6", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014922, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 11.399693761020899, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1055, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:39.170378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014922, "created_at": "2026-09-17T03:12:39.170Z" }, { "sequence": 751, "attempt_id": "89926d21-c595-4a88-b1d9-098ec424d1ba", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 2.999300320050679, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:39.410259+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T03:12:39.410Z" }, { "sequence": 752, "attempt_id": "80db896b-c2d2-406c-9502-6f93c2170291", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006468, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.164799113990739, "model_name": "claude-sonnet-5", "output_tokens": 84, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:39.457109+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006468, "created_at": "2026-09-17T03:12:39.457Z" }, { "sequence": 753, "attempt_id": "7e8ee28b-00d7-49bb-86a7-e517c68a106c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016945, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 3.0609125760383904, "model_name": "claude-opus-5", "output_tokens": 115, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:39.528301+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016945, "created_at": "2026-09-17T03:12:39.528Z" }, { "sequence": 754, "attempt_id": "656d3440-79db-408d-b855-2794839c87ef", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.8361779289552942, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:40.217076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:12:40.217Z" }, { "sequence": 755, "attempt_id": "1929b252-0a72-4e7c-bd9c-8fb147958df2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.7873177449218929, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:40.289198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:12:40.289Z" }, { "sequence": 756, "attempt_id": "b201723f-37c7-42bb-aebb-e687ff0fbf45", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00557, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.5069000920047984, "model_name": "gpt-5.6-terra", "output_tokens": 66, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:40.964398+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00557, "created_at": "2026-09-17T03:12:40.964Z" }, { "sequence": 757, "attempt_id": "30a6a4bc-4144-4721-9baf-aada470f1eab", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.9957898210268468, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:41.342955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003683, "created_at": "2026-09-17T03:12:41.343Z" }, { "sequence": 758, "attempt_id": "6c05c40d-21a9-458c-9f85-fe80de5df438", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00195825, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.6052539580268785, "model_name": "gemini-3.8-flash", "output_tokens": 295, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:41.881969+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00195825, "created_at": "2026-09-17T03:12:41.882Z" }, { "sequence": 759, "attempt_id": "c046b343-8e35-40e0-a095-7e8ba5757f37", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011898, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 8.658658276079223, "model_name": "gemini-3.1-pro-preview", "output_tokens": 799, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:43.227455+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011898, "created_at": "2026-09-17T03:12:43.227Z" }, { "sequence": 760, "attempt_id": "fa5d090c-ccbb-4e74-bc5e-59e78f03d447", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00817625, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 15.296553215011954, "model_name": "muse-spark-1.3", "output_tokens": 1180, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:43.997525+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00817625, "created_at": "2026-09-17T03:12:43.997Z" }, { "sequence": 761, "attempt_id": "ffcfeb96-6a6c-4ffa-a26e-204fe1fbc425", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 5.011076752096415, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:44.228308+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:12:44.228Z" }, { "sequence": 762, "attempt_id": "4d9ff84a-a9db-420b-8ff4-1ad86ee41465", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004956, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.9466904510045424, "model_name": "muse-spark-1.3", "output_tokens": 422, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:44.972888+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004956, "created_at": "2026-09-17T03:12:44.973Z" }, { "sequence": 763, "attempt_id": "249c223c-1f45-4caa-8994-e78e4c3119ac", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.40740028093569, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:45.344816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:12:45.345Z" }, { "sequence": 764, "attempt_id": "77aa6fee-b32a-44d4-82c3-55fc6994f3cb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00604825, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 4.13537210796494, "model_name": "muse-spark-1.2", "output_tokens": 679, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:45.526603+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00604825, "created_at": "2026-09-17T03:12:45.526Z" }, { "sequence": 765, "attempt_id": "f12be6a9-7d69-4bfd-9a36-990d17c7d936", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-003", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 1.2831723470008, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:12:45.578924+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:12:45.579Z" }, { "sequence": 766, "attempt_id": "1181ef1a-6976-4935-8441-7efbb5d599e3", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00606, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.122168380068615, "model_name": "claude-sonnet-5", "output_tokens": 36, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:46.177855+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00606, "created_at": "2026-09-17T03:12:46.178Z" }, { "sequence": 767, "attempt_id": "c7ed21e4-d2ad-4ffe-812a-7bf130468fae", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.497063171933405, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:46.889122+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:12:46.889Z" }, { "sequence": 768, "attempt_id": "f1166305-cf0f-4dfd-8962-5f1ca1a33627", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005964, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.9384014380630106, "model_name": "gpt-5.6-luna", "output_tokens": 94, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:47.565837+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005964, "created_at": "2026-09-17T03:12:47.566Z" }, { "sequence": 769, "attempt_id": "af04311d-8e44-4cae-b4ac-664a4fb1d974", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.6049665519967675, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:47.628989+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:12:47.629Z" }, { "sequence": 770, "attempt_id": "de517669-3e34-445e-b14e-7c4ad6ff6190", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01282, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 8.309079683967866, "model_name": "gemini-3.1-pro-preview", "output_tokens": 879, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:47.881934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01282, "created_at": "2026-09-17T03:12:47.882Z" }, { "sequence": 771, "attempt_id": "eefadfca-6e6d-47ad-b663-870db0771d46", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 5.46353177097626, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:48.749647+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T03:12:48.749Z" }, { "sequence": 772, "attempt_id": "b22a3715-fc8e-4473-9920-9be855ff3ef0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-003", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0066, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.2920627129497007, "model_name": "gpt-5.6-terra", "output_tokens": 147, "prediction": { "choice": "D" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:12:48.871312+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0066, "created_at": "2026-09-17T03:12:48.871Z" }, { "sequence": 773, "attempt_id": "a0ac4348-f3d0-417f-bde7-3748d8cb126a", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00272475, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.2493283089715987, "model_name": "gemini-3.8-flash", "output_tokens": 494, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:12:49.188681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00272475, "created_at": "2026-09-17T03:12:49.188Z" }, { "sequence": 774, "attempt_id": "cab87e6a-4671-4151-b0b7-34e2902af5c2", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.7741560710128397, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:12:49.403500+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003764, "created_at": "2026-09-17T03:12:49.403Z" }, { "sequence": 775, "attempt_id": "60ff5737-8cb3-4f6a-9bc1-ba4a47d433f8", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.5182345689972863, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:49.970243+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T03:12:49.970Z" }, { "sequence": 776, "attempt_id": "29fb646a-a529-4351-9443-d731bad001e7", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005794, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.481073799077421, "model_name": "gemini-3.1-pro-preview", "output_tokens": 289, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:50.706546+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005794, "created_at": "2026-09-17T03:12:50.706Z" }, { "sequence": 777, "attempt_id": "7cacec71-b753-4b19-af67-3d3c92176c2f", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01432, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 1.866024354007095, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:50.784023+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01432, "created_at": "2026-09-17T03:12:50.784Z" }, { "sequence": 778, "attempt_id": "8c1d53ce-12c7-40dd-9434-07608b9e5375", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.18851424602326, "model_name": "claude-sonnet-5", "output_tokens": 75, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:51.425390+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006368, "created_at": "2026-09-17T03:12:51.425Z" }, { "sequence": 779, "attempt_id": "251662de-137b-4af4-9066-f82616720224", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.185379030997865, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:52.204351+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:12:52.204Z" }, { "sequence": 780, "attempt_id": "6ea261ab-fde7-4df1-9d51-9553873a7de7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 0.7689648360246792, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:52.252837+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004944, "created_at": "2026-09-17T03:12:52.253Z" }, { "sequence": 781, "attempt_id": "4660032a-1cfd-4b8d-bfdc-43fe6c6d2361", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.467744673951529, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:52.302707+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005436, "created_at": "2026-09-17T03:12:52.302Z" }, { "sequence": 782, "attempt_id": "48c3b5e7-1bab-43f2-b91e-05ca81c26a36", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.553944182000123, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:52.352706+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T03:12:52.354Z" }, { "sequence": 783, "attempt_id": "c96ac028-d6fd-447b-89da-0d896896c1a5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010632, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.3149995249696076, "model_name": "gpt-5.6-sol", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:53.069822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010632, "created_at": "2026-09-17T03:12:53.070Z" }, { "sequence": 784, "attempt_id": "6a2a5ac7-eb80-45c2-b2d1-44a1fdb67782", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.1127838440006599, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:53.465807+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003555, "created_at": "2026-09-17T03:12:53.465Z" }, { "sequence": 785, "attempt_id": "130d6cfb-b23d-4dee-b7ab-792085634c07", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02066675, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 52.52634105598554, "model_name": "muse-spark-1.3", "output_tokens": 4111, "prediction": { "choice": "A" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:53.777730+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02066675, "created_at": "2026-09-17T03:12:53.777Z" }, { "sequence": 786, "attempt_id": "23c391b9-f734-4a41-9803-12afbccc2e9e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00975225, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.467217574943788, "model_name": "muse-spark-1.2", "output_tokens": 1542, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:55.397740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00975225, "created_at": "2026-09-17T03:12:55.397Z" }, { "sequence": 787, "attempt_id": "952ba353-16b3-4017-9836-b9307feb81f1", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00218625, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 3.331022046972066, "model_name": "gemini-3.8-flash", "output_tokens": 356, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:12:55.630589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00218625, "created_at": "2026-09-17T03:12:55.630Z" }, { "sequence": 788, "attempt_id": "e619c132-adc8-4e98-b159-9cab8beadd0f", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.319595753098838, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:56.153060+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T03:12:56.153Z" }, { "sequence": 789, "attempt_id": "8e5432e5-e2c4-41fd-b734-22ce3dd6b405", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.7621962439734489, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:56.450597+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:12:56.450Z" }, { "sequence": 790, "attempt_id": "319b2749-ac3d-41ea-a0d8-9bfe69abd9f6", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.4885768078966066, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:57.007009+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:12:57.007Z" }, { "sequence": 791, "attempt_id": "6f0c6d12-609d-4323-8749-e12949d67a7e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01803925, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 20.90646203898359, "model_name": "muse-spark-1.2", "output_tokens": 3496, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:57.147930+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01803925, "created_at": "2026-09-17T03:12:57.148Z" }, { "sequence": 792, "attempt_id": "72b27a46-2bd0-45b3-ab40-8fdb8bb3ead7", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9570697660092264, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:57.413717+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T03:12:57.413Z" }, { "sequence": 793, "attempt_id": "20281b94-d007-4a0e-b574-0b48eb8c129d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007802, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.493426387081854, "model_name": "gemini-3.1-pro-preview", "output_tokens": 461, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:57.746567+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007802, "created_at": "2026-09-17T03:12:57.746Z" }, { "sequence": 794, "attempt_id": "0f190af4-3f95-4e58-a892-bb2dbae0e18d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1708207200281322, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:58.379038+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:12:58.379Z" }, { "sequence": 795, "attempt_id": "e32e1fdb-851a-4937-b643-c04685c4bbd5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011292, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.9681470660725608, "model_name": "gpt-5.6-sol", "output_tokens": 81, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:58.470881+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011292, "created_at": "2026-09-17T03:12:58.471Z" }, { "sequence": 796, "attempt_id": "abb8c655-8b49-4e0f-8371-57badbe58331", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.683138520922512, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:58.901707+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:12:58.901Z" }, { "sequence": 797, "attempt_id": "dcf8480f-2f92-4889-9bf1-f86d2661526d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-016", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0057, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.883616484934464, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:12:58.965131+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0057, "created_at": "2026-09-17T03:12:58.965Z" }, { "sequence": 798, "attempt_id": "55e009fb-537a-4cc3-b440-c2938b2bce2d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00761525, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 5.943759222980589, "model_name": "muse-spark-1.2", "output_tokens": 1048, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:59.061666+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00761525, "created_at": "2026-09-17T03:12:59.061Z" }, { "sequence": 799, "attempt_id": "18c529ce-65ad-4d6d-b832-8665b1f9cd49", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-016", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.7385731369722635, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:12:59.191766+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036389999999999995, "created_at": "2026-09-17T03:12:59.191Z" }, { "sequence": 800, "attempt_id": "eda1d68e-7af2-4033-8037-632f9a4c4306", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0037935, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 3.0424181410344318, "model_name": "gemini-3.8-flash", "output_tokens": 779, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:13:00.860867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0037935, "created_at": "2026-09-17T03:13:00.861Z" }, { "sequence": 801, "attempt_id": "7507dc5e-6ff6-4049-ac3d-3ebe13e6ba63", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0131305, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 25.061665587010793, "model_name": "muse-spark-1.3", "output_tokens": 2341, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:00.924286+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0131305, "created_at": "2026-09-17T03:13:00.924Z" }, { "sequence": 802, "attempt_id": "d5cf1a36-6c3f-49a8-8ace-eb1ff3455127", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9789902380434796, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:01.099974+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T03:13:01.100Z" }, { "sequence": 803, "attempt_id": "b5eba811-e7dc-491f-84d2-d0ef78b6bcd2", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00604, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.042343668988906, "model_name": "claude-sonnet-5", "output_tokens": 34, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:01.294678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00604, "created_at": "2026-09-17T03:13:01.294Z" }, { "sequence": 804, "attempt_id": "e15d0845-2ca4-4e61-8fb4-7dc09fb90060", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.6826033380348235, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:01.607176+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:13:01.607Z" }, { "sequence": 805, "attempt_id": "af504d29-b5fb-4af4-b594-a19220f127de", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00661225, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 9.482423731009476, "model_name": "muse-spark-1.3", "output_tokens": 812, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:01.883142+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00661225, "created_at": "2026-09-17T03:13:01.883Z" }, { "sequence": 806, "attempt_id": "e76d7aa3-0804-43d2-b1fc-9f3e5cce9e74", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.6617531690280885, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:02.687695+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:13:02.687Z" }, { "sequence": 807, "attempt_id": "331a4294-78c9-4ff9-a5b6-a5f3cedc4f1e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.8156198459910229, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:02.800036+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:13:02.800Z" }, { "sequence": 808, "attempt_id": "d5267d95-29ae-49fb-878e-12b4850cfbab", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005292, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4787304729688913, "model_name": "gpt-5.6-terra", "output_tokens": 38, "prediction": { "choice": "D" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:13:02.863437+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005292, "created_at": "2026-09-17T03:13:02.865Z" }, { "sequence": 809, "attempt_id": "cac00f55-906b-4909-96f2-1f4347e9ed80", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.7897953540086746, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:02.952300+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:13:02.952Z" }, { "sequence": 810, "attempt_id": "1023c41b-1815-4913-b26d-51db647fc8a7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.3068340088939294, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:03.012067+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:13:03.012Z" }, { "sequence": 811, "attempt_id": "6dc9aa5e-7c91-49b3-bdf9-80c7473a4f91", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.2405230150325224, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:04.104297+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036389999999999995, "created_at": "2026-09-17T03:13:04.104Z" }, { "sequence": 812, "attempt_id": "88d2dc6d-80d2-48c1-b660-b64827a54683", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011998, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 8.154685134999454, "model_name": "gemini-3.1-pro-preview", "output_tokens": 806, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:05.625863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011998, "created_at": "2026-09-17T03:13:05.626Z" }, { "sequence": 813, "attempt_id": "4879b7bd-bcea-4d81-9e32-b665e3bea181", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0065945, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.244312442024238, "model_name": "muse-spark-1.3", "output_tokens": 799, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:05.774396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0065945, "created_at": "2026-09-17T03:13:05.774Z" }, { "sequence": 814, "attempt_id": "fbad34bf-7e4a-425e-be10-63a962c6a479", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010989, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 18.39863368391525, "model_name": "muse-spark-1.3", "output_tokens": 1833, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:06.082628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010989, "created_at": "2026-09-17T03:13:06.082Z" }, { "sequence": 815, "attempt_id": "56144d12-8c78-44b0-afa4-59be7c9c4462", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00952275, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.302882313961163, "model_name": "muse-spark-1.2", "output_tokens": 1488, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:06.268481+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00952275, "created_at": "2026-09-17T03:13:06.268Z" }, { "sequence": 816, "attempt_id": "97afc063-2958-4a48-a90c-09273b0aeaa3", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.7723706429824233, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:06.940068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:13:06.940Z" }, { "sequence": 817, "attempt_id": "aae25dc0-3ad3-47a1-b113-d2a7e1777ad9", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.2078757310518995, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:07.042823+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:13:07.042Z" }, { "sequence": 818, "attempt_id": "3df4a7d6-8ba1-4186-9c40-b06869c90221", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0037785, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.626098091946915, "model_name": "gemini-3.8-flash", "output_tokens": 775, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:13:07.375376+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0037785, "created_at": "2026-09-17T03:13:07.375Z" }, { "sequence": 819, "attempt_id": "67a0c9cf-5dd2-45d0-a04c-88ac2fb39ed9", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7167392029659823, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:07.437927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:13:07.438Z" }, { "sequence": 820, "attempt_id": "dd1dbd97-b5a2-486e-a71e-0cb5489d3ca3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.179823211976327, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:07.510990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:13:07.511Z" }, { "sequence": 821, "attempt_id": "d6384876-fad6-4199-af8a-efd3e5a8d172", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9907499901019037, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:07.991882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:13:07.992Z" }, { "sequence": 822, "attempt_id": "a27fa106-e4cc-4b1a-95ce-2eab6765d79d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9125644250307232, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:08.070240+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:13:08.072Z" }, { "sequence": 823, "attempt_id": "6d5ee063-a3e6-478c-b993-feebf46c3025", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007846, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 6.1909040089230984, "model_name": "gemini-3.1-pro-preview", "output_tokens": 460, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:08.131298+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007846, "created_at": "2026-09-17T03:13:08.131Z" }, { "sequence": 824, "attempt_id": "0e2e88a1-0749-4640-a1bf-242bab8286c1", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.950238851015456, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:08.189195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:13:08.189Z" }, { "sequence": 825, "attempt_id": "c2d9b039-fb99-4f4b-b3a0-afdc8596dc67", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 5.662840030970983, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:08.737005+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:13:08.737Z" }, { "sequence": 826, "attempt_id": "ef635988-b4f8-4206-a150-0035ed736e56", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.392915635020472, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:08.954954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:13:08.955Z" }, { "sequence": 827, "attempt_id": "eae29c9f-409c-47c2-8b22-96ccd1c85b5e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.66689288103953, "model_name": "gemini-3.8-flash", "output_tokens": 107, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:09.158630+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00125175, "created_at": "2026-09-17T03:13:09.158Z" }, { "sequence": 828, "attempt_id": "9ceb5779-4ca7-4ccc-9739-08579fa087a7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040355, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.5389962319750339, "model_name": "muse-spark-1.2", "output_tokens": 206, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:09.670666+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0040355, "created_at": "2026-09-17T03:13:09.670Z" }, { "sequence": 829, "attempt_id": "388469e9-763a-46e9-8724-a980099cfcc8", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-003", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.8236723990412429, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:09.840486+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001884, "created_at": "2026-09-17T03:13:09.840Z" }, { "sequence": 830, "attempt_id": "bad0c73b-aaac-4987-8560-d88eca842c06", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00368275, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.772790111019276, "model_name": "muse-spark-1.3", "output_tokens": 123, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:09.898239+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00368275, "created_at": "2026-09-17T03:13:09.900Z" }, { "sequence": 831, "attempt_id": "de15e280-c6fd-47ff-94dc-9a6295edb007", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.021994363050908, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:11.241941+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02484, "created_at": "2026-09-17T03:13:11.242Z" }, { "sequence": 832, "attempt_id": "fe1ce97c-64a9-4285-8b8b-0da98252d0d0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010996, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6384416819782928, "model_name": "gpt-5.6-sol", "output_tokens": 67, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:11.372702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010996, "created_at": "2026-09-17T03:13:11.372Z" }, { "sequence": 833, "attempt_id": "636da606-1520-4ff6-be18-0a380009b926", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004764, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 4.101847501005977, "model_name": "gemini-3.1-pro-preview", "output_tokens": 208, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:11.541127+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004764, "created_at": "2026-09-17T03:13:11.541Z" }, { "sequence": 834, "attempt_id": "edbacf19-7103-464d-9725-0a56e8d236d4", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018465, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.4543996149441227, "model_name": "claude-opus-5", "output_tokens": 165, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:11.704631+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.018465, "created_at": "2026-09-17T03:13:11.704Z" }, { "sequence": 835, "attempt_id": "90528f56-072b-4dc8-899d-aff7b0c3848d", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006886, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.0162721569649875, "model_name": "claude-sonnet-5", "output_tokens": 115, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:11.817050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006886, "created_at": "2026-09-17T03:13:11.817Z" }, { "sequence": 836, "attempt_id": "a0fe2fd8-0b94-43d2-b56b-f50e26deebd1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005692000000000001, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.0974180650664493, "model_name": "gpt-5.6-luna", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:12.056452+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005692000000000001, "created_at": "2026-09-17T03:13:12.057Z" }, { "sequence": 837, "attempt_id": "9332c05f-f1fd-4dcf-93e0-9b8d2660de80", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005668, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.2154372490476817, "model_name": "gpt-5.6-terra", "output_tokens": 70, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:12.115680+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005668, "created_at": "2026-09-17T03:13:12.115Z" }, { "sequence": 838, "attempt_id": "fb4e3c7b-179a-4253-a468-0382753fb8ca", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0294, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 4.3097757620271295, "model_name": "claude-fable-5-1", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:12.556598+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0294, "created_at": "2026-09-17T03:13:12.556Z" }, { "sequence": 839, "attempt_id": "10f2354b-c6ad-4342-86a9-f88b76e37054", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 2.183314824011177, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:14.358729+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01421, "created_at": "2026-09-17T03:13:14.358Z" }, { "sequence": 840, "attempt_id": "1335878e-14ec-4b1b-b75b-c231e17f76ce", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.9402421499835327, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:14.550772+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005684, "created_at": "2026-09-17T03:13:14.550Z" }, { "sequence": 841, "attempt_id": "66016aed-1d83-439d-b52b-8b2c99d5a2c4", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00374775, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 3.333822738029994, "model_name": "gemini-3.8-flash", "output_tokens": 767, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:14.766801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00374775, "created_at": "2026-09-17T03:13:14.766Z" }, { "sequence": 842, "attempt_id": "f2459041-8bf9-4e0e-bba4-b462e0f533d6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.6471397160785273, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:15.060826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001833, "created_at": "2026-09-17T03:13:15.061Z" }, { "sequence": 843, "attempt_id": "b44deff2-275b-4fd9-b6a3-a3903c21c900", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 3.225401822011918, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:15.342070+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02844, "created_at": "2026-09-17T03:13:15.342Z" }, { "sequence": 844, "attempt_id": "d9f0dae2-f77e-43a1-a9af-fdee2648dce8", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036359999999999995, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 4.047107063001022, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:15.650337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036359999999999995, "created_at": "2026-09-17T03:13:15.650Z" }, { "sequence": 845, "attempt_id": "e8c77810-df88-4f74-9c84-76d9683c1677", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.9671448599547148, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:16.086132+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004904, "created_at": "2026-09-17T03:13:16.086Z" }, { "sequence": 846, "attempt_id": "0e33d3ad-0a82-444b-b54a-cac23d09618b", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007016, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 4.8384716040454805, "model_name": "gemini-3.1-pro-preview", "output_tokens": 391, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:16.148979+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007016, "created_at": "2026-09-17T03:13:16.149Z" }, { "sequence": 847, "attempt_id": "008335fe-4209-4060-94cb-264c62115f93", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.6729174259817228, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:16.490039+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009752, "created_at": "2026-09-17T03:13:16.490Z" }, { "sequence": 848, "attempt_id": "0fb6267c-8d79-4153-8425-ce8f699b708c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.106917590019293, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:16.710784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02438, "created_at": "2026-09-17T03:13:16.710Z" }, { "sequence": 849, "attempt_id": "df0d1d67-ff36-494d-a62a-eb8c87ab0ee9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.9997370389755815, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:17.209274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:13:17.209Z" }, { "sequence": 850, "attempt_id": "f6a69760-e1b5-4e05-a1dc-3de80f461374", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.2334497509291396, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:17.635852+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004904, "created_at": "2026-09-17T03:13:17.636Z" }, { "sequence": 851, "attempt_id": "b6a3f37e-3147-49d3-9aaf-fb4ee851dc31", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012075, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.6746932459063828, "model_name": "gemini-3.8-flash", "output_tokens": 99, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:17.824034+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0012075, "created_at": "2026-09-17T03:13:17.824Z" }, { "sequence": 852, "attempt_id": "59ef29b3-8fa6-4602-b604-3482f2b9c99d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00362925, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.4851508410647511, "model_name": "muse-spark-1.2", "output_tokens": 116, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:18.266653+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00362925, "created_at": "2026-09-17T03:13:18.266Z" }, { "sequence": 853, "attempt_id": "352af212-5bb8-4dce-ba4f-35796b6c354b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003948, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.9605802999576554, "model_name": "muse-spark-1.3", "output_tokens": 191, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:18.511323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003948, "created_at": "2026-09-17T03:13:18.511Z" }, { "sequence": 854, "attempt_id": "cb1a30d4-ef8d-4186-b6af-38a4aeacee95", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004486, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 3.5737783110234886, "model_name": "gemini-3.1-pro-preview", "output_tokens": 188, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:19.277555+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004486, "created_at": "2026-09-17T03:13:19.277Z" }, { "sequence": 855, "attempt_id": "e761fe2b-207d-4c0e-b95c-bb1fb819ef79", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.1579677530098706, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:19.486395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:13:19.486Z" }, { "sequence": 856, "attempt_id": "86158311-62a4-45c1-8fba-892bfc6ea6d7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00891975, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 8.210171347018331, "model_name": "muse-spark-1.2", "output_tokens": 1347, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:20.093668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00891975, "created_at": "2026-09-17T03:13:20.093Z" }, { "sequence": 857, "attempt_id": "f6f9603d-9db0-4c8b-ad38-7ed9df90f7c7", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006638, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.246818169951439, "model_name": "claude-sonnet-5", "output_tokens": 96, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:20.156066+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006638, "created_at": "2026-09-17T03:13:20.156Z" }, { "sequence": 858, "attempt_id": "aca3045d-434a-4190-b86e-6b2a955f0646", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.405922662001103, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:20.217239+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01447, "created_at": "2026-09-17T03:13:20.217Z" }, { "sequence": 859, "attempt_id": "8bd1cba4-3ba7-4ea1-a85c-b617bbccd18c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00603825, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 8.687176381004974, "model_name": "muse-spark-1.3", "output_tokens": 669, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:20.452183+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00603825, "created_at": "2026-09-17T03:13:20.452Z" }, { "sequence": 860, "attempt_id": "9ecba59a-babf-4176-a027-1e76edf0784d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9064801159547642, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:20.513442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:13:20.513Z" }, { "sequence": 861, "attempt_id": "5e5cba92-680f-4ac8-8b46-40292122afcf", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.319516382063739, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:20.590177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:13:20.590Z" }, { "sequence": 862, "attempt_id": "89a825a1-72b5-4672-a774-9953ab8161fd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.4276892970083281, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:20.773640+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:13:20.773Z" }, { "sequence": 863, "attempt_id": "8c588467-a513-4870-bfca-44fada68143e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.0732390220509842, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:21.587069+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:13:21.587Z" }, { "sequence": 864, "attempt_id": "931b2dfc-8bfe-4715-be98-af6ee267f639", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005556, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.050975520047359, "model_name": "gpt-5.6-terra", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:21.653484+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005556, "created_at": "2026-09-17T03:13:21.653Z" }, { "sequence": 865, "attempt_id": "2184244a-53ac-4ae8-b836-90eed28dc4b5", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0019575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.1251641099806875, "model_name": "gemini-3.8-flash", "output_tokens": 292, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:22.405130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0019575, "created_at": "2026-09-17T03:13:22.405Z" }, { "sequence": 866, "attempt_id": "635d005f-4ca3-4bc4-9112-4a35ac0bad72", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005604, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.550117710023187, "model_name": "gpt-5.6-luna", "output_tokens": 67, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:22.706554+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005604, "created_at": "2026-09-17T03:13:22.706Z" }, { "sequence": 867, "attempt_id": "c31e4a0b-cf6f-414a-a899-34427773626d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01938275, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 20.699035033932887, "model_name": "muse-spark-1.2", "output_tokens": 3808, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:23.711501+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01938275, "created_at": "2026-09-17T03:13:23.711Z" }, { "sequence": 868, "attempt_id": "1847d5e7-5514-4b6a-b490-80863f72ff93", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.339963307022117, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:23.804496+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:13:23.804Z" }, { "sequence": 869, "attempt_id": "40ff78ae-c6ee-493a-b498-25f14eb9557e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007138, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.438010416110046, "model_name": "claude-sonnet-5", "output_tokens": 127, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:24.156897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007138, "created_at": "2026-09-17T03:13:24.157Z" }, { "sequence": 870, "attempt_id": "e0e64be8-b866-445f-806c-b5bad2180ece", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016795, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.698781384038739, "model_name": "claude-opus-5", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:24.352638+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016795, "created_at": "2026-09-17T03:13:24.352Z" }, { "sequence": 871, "attempt_id": "a90ac467-1449-40a1-b309-da42d8073020", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005528, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.312374165048823, "model_name": "gemini-3.1-pro-preview", "output_tokens": 269, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:24.588935+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005528, "created_at": "2026-09-17T03:13:24.589Z" }, { "sequence": 872, "attempt_id": "c3f2c1b4-b2f3-43eb-b73f-14ad5dea7d5c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.913281961926259, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:24.681929+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:13:24.682Z" }, { "sequence": 873, "attempt_id": "37318aeb-df51-4747-90a9-7a7594dda48a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005603, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.116557617904618, "model_name": "muse-spark-1.2", "output_tokens": 571, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:24.765696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005603, "created_at": "2026-09-17T03:13:24.766Z" }, { "sequence": 874, "attempt_id": "8dd1e161-309a-4d93-8f36-c0fb7d438557", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006012000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.5082583329640329, "model_name": "gpt-5.6-luna", "output_tokens": 90, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:25.727456+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006012000000000001, "created_at": "2026-09-17T03:13:25.727Z" }, { "sequence": 875, "attempt_id": "6f3b741b-d367-4421-93ef-b8fd4bb15bba", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.930992565001361, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:25.803229+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:13:25.803Z" }, { "sequence": 876, "attempt_id": "2989d7c1-7d1f-48a3-bfe4-0933cd413ccd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005856, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.9969723470276222, "model_name": "gpt-5.6-terra", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:25.863926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005856, "created_at": "2026-09-17T03:13:25.864Z" }, { "sequence": 877, "attempt_id": "9a52e050-31ec-4084-a0ea-f4638b23e2b8", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.1800083990674466, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:25.926932+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:13:25.927Z" }, { "sequence": 878, "attempt_id": "e05a1a51-dcb5-4b1e-9308-97e7410906f0", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00185475, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.1034698580624536, "model_name": "gemini-3.8-flash", "output_tokens": 252, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:26.756189+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00185475, "created_at": "2026-09-17T03:13:26.756Z" }, { "sequence": 879, "attempt_id": "50eb887a-d34a-4dc4-9bfe-2d65133e782d", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.7038172039901838, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:27.524498+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T03:13:27.524Z" }, { "sequence": 880, "attempt_id": "225dcfaa-5b39-43cb-96a0-c2d1060a553e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011344, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.017760818009265, "model_name": "gpt-5.6-sol", "output_tokens": 74, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:27.792224+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011344, "created_at": "2026-09-17T03:13:27.792Z" }, { "sequence": 881, "attempt_id": "b43822bd-8dde-42c4-9b3b-36223782642c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006448, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.7233662490034476, "model_name": "claude-sonnet-5", "output_tokens": 80, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:28.712552+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006448, "created_at": "2026-09-17T03:13:28.712Z" }, { "sequence": 882, "attempt_id": "79805876-4f6e-4381-a17b-84493cab3e53", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01557, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.7952971711056307, "model_name": "claude-opus-5", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:28.777109+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01557, "created_at": "2026-09-17T03:13:28.779Z" }, { "sequence": 883, "attempt_id": "92db5a95-c1d1-4d2d-9735-6fac7eb0f007", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00593025, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 8.583682957920246, "model_name": "muse-spark-1.3", "output_tokens": 648, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:29.157958+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00593025, "created_at": "2026-09-17T03:13:29.158Z" }, { "sequence": 884, "attempt_id": "3b7785cf-f581-4fc2-93dd-1fc9084d838a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.330507531063631, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:29.221588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T03:13:29.221Z" }, { "sequence": 885, "attempt_id": "fb4c382d-cec2-4acd-9d82-38d07c775e48", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.6935948159079999, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:29.284205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:13:29.284Z" }, { "sequence": 886, "attempt_id": "1894187f-20a1-4add-94f7-29fc1bdfbe42", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.6916830010013655, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:29.547851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:13:29.548Z" }, { "sequence": 887, "attempt_id": "b75cafc6-1d6d-4dcf-944b-2be8265453d0", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006119, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 3.8893353029852733, "model_name": "muse-spark-1.2", "output_tokens": 673, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:29.692857+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006119, "created_at": "2026-09-17T03:13:29.693Z" }, { "sequence": 888, "attempt_id": "fb723871-ae1c-4bfc-a8ca-62b7016c1b3d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.0543499920750037, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:29.895252+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:13:29.895Z" }, { "sequence": 889, "attempt_id": "98cc3bf9-34a6-4d36-9ba3-eb21fd64a892", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005404, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.2770520530175418, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:30.054420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005404, "created_at": "2026-09-17T03:13:30.054Z" }, { "sequence": 890, "attempt_id": "efa7138c-0b8b-40a3-a08c-0f589e30bd01", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00566425, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 5.412234788062051, "model_name": "muse-spark-1.3", "output_tokens": 566, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:30.238047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00566425, "created_at": "2026-09-17T03:13:30.238Z" }, { "sequence": 891, "attempt_id": "d286565e-914a-479a-9493-ec304484d6a0", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.2381884990027174, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:13:30.584512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003707, "created_at": "2026-09-17T03:13:30.584Z" }, { "sequence": 892, "attempt_id": "3b43fc1f-ddba-4419-9e71-72b6d94107d9", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00158175, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.8479789099656045, "model_name": "gemini-3.8-flash", "output_tokens": 193, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:31.132577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00158175, "created_at": "2026-09-17T03:13:31.132Z" }, { "sequence": 893, "attempt_id": "4f5fd9d2-649f-440f-9e0e-1278e2083007", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-023", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.8302049450576305, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:31.467942+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:13:31.468Z" }, { "sequence": 894, "attempt_id": "3efdaa4b-5cc1-4022-b59b-374d37ad56e2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0043625, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 1.8104202680988237, "model_name": "muse-spark-1.2", "output_tokens": 280, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:31.567450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0043625, "created_at": "2026-09-17T03:13:31.567Z" }, { "sequence": 895, "attempt_id": "1e2dd5f0-2f36-4f2a-bcdc-9cd7a459d948", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003812, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 3.312720200046897, "model_name": "gemini-3.1-pro-preview", "output_tokens": 127, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:32.534703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003812, "created_at": "2026-09-17T03:13:32.534Z" }, { "sequence": 896, "attempt_id": "83f72620-c387-4234-a843-cac6c1a5c35e", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01247, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.248653414077125, "model_name": "gemini-3.1-pro-preview", "output_tokens": 837, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:32.661680+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01247, "created_at": "2026-09-17T03:13:32.661Z" }, { "sequence": 897, "attempt_id": "0df9d83f-47e1-48d9-bba8-b6c626d0e51e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007178, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.8761821819934994, "model_name": "claude-sonnet-5", "output_tokens": 131, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:33.176425+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007178, "created_at": "2026-09-17T03:13:33.176Z" }, { "sequence": 898, "attempt_id": "eeef3f86-6855-4785-b6be-275f739f6477", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00467275, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.728388553019613, "model_name": "muse-spark-1.3", "output_tokens": 353, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:33.340030+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00467275, "created_at": "2026-09-17T03:13:33.340Z" }, { "sequence": 899, "attempt_id": "67525280-1f0a-4710-978b-0e73cdf3f5f5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005796, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7309958960395306, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:33.403416+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005796, "created_at": "2026-09-17T03:13:33.403Z" }, { "sequence": 900, "attempt_id": "67e7b8a2-b2ef-46d2-91b2-7dcdf081c72f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011284, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.093808191944845, "model_name": "gpt-5.6-sol", "output_tokens": 71, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:33.627217+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011284, "created_at": "2026-09-17T03:13:33.627Z" }, { "sequence": 901, "attempt_id": "98264197-d90a-4d74-a980-a939e5f28ad8", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02726, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.668497856007889, "model_name": "gpt-6-astra", "output_tokens": 52, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:33.863140+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02726, "created_at": "2026-09-17T03:13:33.863Z" }, { "sequence": 902, "attempt_id": "e7535979-a51a-4098-ab7d-7e3926637d1d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-023", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7518500150181353, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:34.155582+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003789, "created_at": "2026-09-17T03:13:34.155Z" }, { "sequence": 903, "attempt_id": "6396c24a-522f-4efd-a14d-7cc31cfc23ef", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005904, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.8220316580263898, "model_name": "gpt-5.6-luna", "output_tokens": 81, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:34.420882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005904, "created_at": "2026-09-17T03:13:34.421Z" }, { "sequence": 904, "attempt_id": "9e136342-4843-480a-9d9f-b90f4565b41c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01972, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 4.604339350946248, "model_name": "claude-opus-5", "output_tokens": 202, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:34.720538+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01972, "created_at": "2026-09-17T03:13:34.720Z" }, { "sequence": 905, "attempt_id": "17768288-de65-4ea0-8059-f1cef7598819", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.7383290559519082, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:35.529006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:13:35.529Z" }, { "sequence": 906, "attempt_id": "942b72a4-1719-4d1d-88b3-a19793b20162", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00258975, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.309056355035864, "model_name": "gemini-3.8-flash", "output_tokens": 448, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:13:35.602116+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00258975, "created_at": "2026-09-17T03:13:35.603Z" }, { "sequence": 907, "attempt_id": "ab5ca286-a6b1-4739-8890-65abe7b5285c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.738941483083181, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:36.227991+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:13:36.228Z" }, { "sequence": 908, "attempt_id": "cde603af-b870-4ba2-aa9f-1b613c4040a5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.394972661975771, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:36.616073+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:13:36.616Z" }, { "sequence": 909, "attempt_id": "9fc02156-0325-4a4b-9ce4-a7dba938590c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-023", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.03001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 6.725144563009962, "model_name": "claude-fable-5-1", "output_tokens": 13, "prediction": { "choice": "B" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:36.683708+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03001, "created_at": "2026-09-17T03:13:36.683Z" }, { "sequence": 910, "attempt_id": "e547b321-166b-4ae0-8bce-3c09b2aff159", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0275444189319387, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:37.321003+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:13:37.321Z" }, { "sequence": 911, "attempt_id": "3c1a97f2-7a28-440d-876f-cfed5d804bd9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7652569300262257, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:37.432193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:13:37.432Z" }, { "sequence": 912, "attempt_id": "23b693a4-413e-4e1a-895c-ad3c7700ba57", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.9692281269235536, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:37.571633+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:13:37.571Z" }, { "sequence": 913, "attempt_id": "6e64f08b-0ed0-4616-ba2a-cb6bf91ca34b", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.0481992569984868, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:13:38.556517+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:13:38.556Z" }, { "sequence": 914, "attempt_id": "460f5f4b-a4c5-49da-8aa3-4a29c1279cb8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009674, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.910796329029836, "model_name": "gemini-3.1-pro-preview", "output_tokens": 604, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:38.646582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009674, "created_at": "2026-09-17T03:13:38.646Z" }, { "sequence": 915, "attempt_id": "cb3d59f6-7efa-43a6-bf75-fb0dce089254", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00119775, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.2946618710411713, "model_name": "gemini-3.8-flash", "output_tokens": 96, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:38.713531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00119775, "created_at": "2026-09-17T03:13:38.713Z" }, { "sequence": 916, "attempt_id": "5ebcf80e-d466-41d6-8202-df64d7d93737", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.7311101240338758, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:39.412626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:13:39.412Z" }, { "sequence": 917, "attempt_id": "61c9ee67-cf11-4341-906a-9e55ed09ad62", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 6.0935818259604275, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:40.020525+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:13:40.020Z" }, { "sequence": 918, "attempt_id": "666d7680-910e-4e23-82a9-51729f175c38", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004027, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.5149732339195907, "model_name": "muse-spark-1.2", "output_tokens": 209, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:40.137028+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004027, "created_at": "2026-09-17T03:13:40.137Z" }, { "sequence": 919, "attempt_id": "7b2f3a69-056e-4e45-b86e-1dc9ff0b633c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003614, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.409564721980132, "model_name": "gemini-3.1-pro-preview", "output_tokens": 115, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:40.204902+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003614, "created_at": "2026-09-17T03:13:40.206Z" }, { "sequence": 920, "attempt_id": "d8772856-610c-4268-8da1-d7a5f8737345", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.8608528899494559, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:40.637959+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015025, "created_at": "2026-09-17T03:13:40.638Z" }, { "sequence": 921, "attempt_id": "ac9895bc-1899-4bd1-909f-0c220c47cc58", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6763078770600259, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:40.762333+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T03:13:40.762Z" }, { "sequence": 922, "attempt_id": "1e0a8fff-6ebb-4bcf-84d8-5d9d553a0ebc", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.101820612908341, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:41.385702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:13:41.385Z" }, { "sequence": 923, "attempt_id": "6daebce5-1472-469a-baca-2c71706e17c9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004792, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.879870869917795, "model_name": "muse-spark-1.3", "output_tokens": 389, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:41.515596+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004792, "created_at": "2026-09-17T03:13:41.515Z" }, { "sequence": 924, "attempt_id": "f491e8d3-47e8-48f8-b45a-22895aa69d9c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.170659836032428, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:41.651450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00601, "created_at": "2026-09-17T03:13:41.651Z" }, { "sequence": 925, "attempt_id": "506894e5-08b2-4f9f-861b-3757819f3a74", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005418, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.3410958349704742, "model_name": "gpt-5.6-terra", "output_tokens": 41, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:42.046965+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005418, "created_at": "2026-09-17T03:13:42.047Z" }, { "sequence": 926, "attempt_id": "681a6f6e-3d19-4dc9-afb3-09ce1779b9c8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005466000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.6278058440657333, "model_name": "gpt-5.6-luna", "output_tokens": 45, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:42.453619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005466000000000001, "created_at": "2026-09-17T03:13:42.453Z" }, { "sequence": 927, "attempt_id": "3316e605-d919-4d6c-bbe7-c41deb83d4bd", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.8670966729987413, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:13:42.581668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:13:42.581Z" }, { "sequence": 928, "attempt_id": "f901ed13-11a2-45a3-84cf-69463f6fa708", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.3881495830137283, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:42.647178+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:13:42.648Z" }, { "sequence": 929, "attempt_id": "c75ee848-67e2-4a0c-bb6d-0ae5a2db78c4", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.27214603300672, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:42.986003+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:13:42.986Z" }, { "sequence": 930, "attempt_id": "be53e034-6626-48ec-b54a-10bcc53cf150", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00178125, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.6200488790636882, "model_name": "gemini-3.8-flash", "output_tokens": 233, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:43.199123+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00178125, "created_at": "2026-09-17T03:13:43.199Z" }, { "sequence": 931, "attempt_id": "82d5187f-90c2-4441-8c5b-540eb85cae2b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01114675, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.147914691013284, "model_name": "muse-spark-1.2", "output_tokens": 1856, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:43.839308+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01114675, "created_at": "2026-09-17T03:13:43.839Z" }, { "sequence": 932, "attempt_id": "c76671e5-47d9-4320-9ac4-660c46810308", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6794085720321164, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:43.943264+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:13:43.943Z" }, { "sequence": 933, "attempt_id": "90ab0a6d-5818-4209-a96b-2689f00a7196", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.9262015679851174, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:44.638797+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:13:44.639Z" }, { "sequence": 934, "attempt_id": "f9dbedd2-68fc-4b60-9d80-862e7923779f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6559726300183684, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:44.711823+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:13:44.715Z" }, { "sequence": 935, "attempt_id": "e2ce611d-24b7-4e92-a786-ce82e93d4a4d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00454825, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.188646031077951, "model_name": "muse-spark-1.2", "output_tokens": 304, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:44.781853+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00454825, "created_at": "2026-09-17T03:13:44.781Z" }, { "sequence": 936, "attempt_id": "c6db8330-ea42-4b74-9f9b-e3fd789339a1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1389097860082984, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:45.147543+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:13:45.147Z" }, { "sequence": 937, "attempt_id": "5b18b876-cd1f-4f7c-8d74-6b7d49431938", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004796, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.88671302201692, "model_name": "gemini-3.1-pro-preview", "output_tokens": 198, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:45.349394+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004796, "created_at": "2026-09-17T03:13:45.349Z" }, { "sequence": 938, "attempt_id": "ba95290d-9fae-44d2-8ddf-e6c140986298", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7059440129669383, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:45.611693+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:13:45.611Z" }, { "sequence": 939, "attempt_id": "7d80d2cc-f65b-4de8-a78b-8db78e647bdf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.10583849507384, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:45.817941+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:13:45.818Z" }, { "sequence": 940, "attempt_id": "69a76771-101b-4442-b5c6-728da82f995f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.255968836019747, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:45.903416+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:13:45.903Z" }, { "sequence": 941, "attempt_id": "202072cf-b29c-4144-b93b-563d2f93fd1e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.0171194869326428, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:13:46.434506+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:13:46.434Z" }, { "sequence": 942, "attempt_id": "4d17f5fb-2a40-4022-a05b-5e21d0c98df4", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049945, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.5344023630023, "model_name": "muse-spark-1.3", "output_tokens": 409, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:46.647213+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049945, "created_at": "2026-09-17T03:13:46.647Z" }, { "sequence": 943, "attempt_id": "c793ed6d-8cf2-4dde-98b1-e48d08850881", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00131025, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.4418715180363506, "model_name": "gemini-3.8-flash", "output_tokens": 122, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:46.708147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00131025, "created_at": "2026-09-17T03:13:46.709Z" }, { "sequence": 944, "attempt_id": "e2f85fc0-a84f-4229-ba80-d1fda09f1146", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0230165, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 44.137095592101105, "model_name": "muse-spark-1.3", "output_tokens": 4663, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:47.064068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0230165, "created_at": "2026-09-17T03:13:47.064Z" }, { "sequence": 945, "attempt_id": "bed17221-e443-4481-8fc7-30227aba4ac8", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.8103386390721425, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:47.585198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T03:13:47.585Z" }, { "sequence": 946, "attempt_id": "704ae4a8-624e-46b6-b4c9-a03610f860a3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.868992252042517, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:47.653842+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:13:47.655Z" }, { "sequence": 947, "attempt_id": "53d3ad5e-c1aa-431b-99d5-7134c3f9fbaf", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003906, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.2383609699318185, "model_name": "gemini-3.1-pro-preview", "output_tokens": 136, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:13:48.086657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003906, "created_at": "2026-09-17T03:13:48.086Z" }, { "sequence": 948, "attempt_id": "ba270137-aac0-4075-909e-8808c9a3383b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.0345410130685195, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:48.536981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T03:13:48.537Z" }, { "sequence": 949, "attempt_id": "b1292de1-2424-4570-b2e5-04eb46dfe071", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.8390053869225085, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:48.604774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:13:48.606Z" }, { "sequence": 950, "attempt_id": "f69cb93a-85e7-46e8-839f-b17b683184bc", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0043835, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.9603804970392957, "model_name": "muse-spark-1.3", "output_tokens": 287, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:48.678555+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0043835, "created_at": "2026-09-17T03:13:48.678Z" }, { "sequence": 951, "attempt_id": "77307a4d-52a2-49c7-8f86-9bfcb49b9972", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.9973100199131295, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:48.744063+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:13:48.744Z" }, { "sequence": 952, "attempt_id": "3a5f77c4-504e-46d2-a26e-aee0b0314c7a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.7390963530633599, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:48.894908+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:13:48.895Z" }, { "sequence": 953, "attempt_id": "1b32794d-5ae6-4ca7-b45f-4abb9a5ce171", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.8439021781086922, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:48.975580+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:13:48.975Z" }, { "sequence": 954, "attempt_id": "b748f005-1c70-4b94-b6e5-000bf9b7269a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.556522549013607, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:49.510357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T03:13:49.510Z" }, { "sequence": 955, "attempt_id": "f49a2a8a-2dcb-4528-a559-fff579d4310c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00609625, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.8690432839794084, "model_name": "muse-spark-1.2", "output_tokens": 690, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:49.740323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00609625, "created_at": "2026-09-17T03:13:49.740Z" }, { "sequence": 956, "attempt_id": "61862baf-4eb7-4053-8f73-6c64c90697ce", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.0426319680409506, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:49.812789+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:13:49.813Z" }, { "sequence": 957, "attempt_id": "2c3c390b-55cc-4aec-87ea-4613afabc73c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00407425, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.1305083750048652, "model_name": "muse-spark-1.2", "output_tokens": 221, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:50.090522+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00407425, "created_at": "2026-09-17T03:13:50.090Z" }, { "sequence": 958, "attempt_id": "83b0c289-2f88-4afe-9e5e-82acf8a0881b", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.6856301290681586, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:50.564221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:13:50.564Z" }, { "sequence": 959, "attempt_id": "5d9c095e-6b45-4c8b-9e5f-81dcbec0c42a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 3.10067811596673, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:50.754810+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:13:50.754Z" }, { "sequence": 960, "attempt_id": "822a3059-2c10-4ae6-a51c-dff810554e3b", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00119175, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 2.6007460989058018, "model_name": "gemini-3.8-flash", "output_tokens": 95, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:51.279719+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00119175, "created_at": "2026-09-17T03:13:51.279Z" }, { "sequence": 961, "attempt_id": "eded5bd3-c37a-432b-8fcb-b34dc5efb65e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9246367890154943, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:51.505559+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:13:51.505Z" }, { "sequence": 962, "attempt_id": "8750baa6-202e-4ecf-9668-906b272ae6f5", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.8165542220231146, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:51.973964+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:13:51.974Z" }, { "sequence": 963, "attempt_id": "064ec666-9539-4ddb-93e1-a70de22ec95c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.351972009986639, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:52.042820+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:13:52.043Z" }, { "sequence": 964, "attempt_id": "257963dd-c68d-498d-9788-39f3c5c30804", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.2412164049455896, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:52.123401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:13:52.123Z" }, { "sequence": 965, "attempt_id": "6d245191-f1d2-4455-8505-8086ee393820", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005088, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.239227335085161, "model_name": "gpt-5.6-terra", "output_tokens": 29, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:52.188477+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005088, "created_at": "2026-09-17T03:13:52.188Z" }, { "sequence": 966, "attempt_id": "00b1f43e-2fd3-43c1-aad0-0b0c95128279", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2171014670748264, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:52.565706+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:13:52.565Z" }, { "sequence": 967, "attempt_id": "26ab3eac-d047-4d7c-88d9-4f072df6df54", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005528, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.986104427021928, "model_name": "gemini-3.1-pro-preview", "output_tokens": 275, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:52.638246+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005528, "created_at": "2026-09-17T03:13:52.638Z" }, { "sequence": 968, "attempt_id": "1f29e88b-1101-40a7-ba77-40d7dcd95a8e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.586402195971459, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:52.702692+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:13:52.702Z" }, { "sequence": 969, "attempt_id": "b3f19c47-0178-4601-aee0-422950ef3553", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0046395, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.891973796999082, "model_name": "muse-spark-1.3", "output_tokens": 354, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:52.769139+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0046395, "created_at": "2026-09-17T03:13:52.769Z" }, { "sequence": 970, "attempt_id": "278b425c-664b-415f-93ca-23e76651f6a3", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.9243453930830583, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:13:53.048120+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:13:53.048Z" }, { "sequence": 971, "attempt_id": "9006aaad-b5c5-4270-abf6-ac930f03e44c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0035, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.0807970019523054, "model_name": "muse-spark-1.2", "output_tokens": 85, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:53.336523+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0035, "created_at": "2026-09-17T03:13:53.336Z" }, { "sequence": 972, "attempt_id": "b539957a-2664-4bbb-ba28-9efd9f5024e1", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001329, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.3216303659137338, "model_name": "gemini-3.8-flash", "output_tokens": 131, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:53.406362+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001329, "created_at": "2026-09-17T03:13:53.408Z" }, { "sequence": 973, "attempt_id": "fa4ffe96-a9e6-4efb-a19f-ff62932f5888", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.1420170350465924, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:53.979854+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:13:53.980Z" }, { "sequence": 974, "attempt_id": "616a4475-a104-456f-93d6-ed0356962a36", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003638, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.7974426470464095, "model_name": "gemini-3.1-pro-preview", "output_tokens": 117, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:54.373272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003638, "created_at": "2026-09-17T03:13:54.373Z" }, { "sequence": 975, "attempt_id": "fdc57565-81da-406b-95b1-e9a4c58deeeb", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005978, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 1.8207180729368702, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "F" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:13:54.590273+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005978, "created_at": "2026-09-17T03:13:54.590Z" }, { "sequence": 976, "attempt_id": "0a46231f-cbd7-4a70-ad74-103de28b0501", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005592, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7651015940355137, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:13:55.242972+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005592, "created_at": "2026-09-17T03:13:55.243Z" }, { "sequence": 977, "attempt_id": "367803f7-7ca3-4a8a-a399-2145d31b1a0a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02701, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.564992411993444, "model_name": "gpt-6-astra", "output_tokens": 47, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:13:55.680217+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02701, "created_at": "2026-09-17T03:13:55.680Z" }, { "sequence": 978, "attempt_id": "b563c5bd-45ca-4f3b-baaa-fe7e04f183ce", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00449875, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.7361105909803882, "model_name": "muse-spark-1.3", "output_tokens": 320, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:55.989557+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00449875, "created_at": "2026-09-17T03:13:55.989Z" }, { "sequence": 979, "attempt_id": "1450038a-16dc-4a83-a5e5-3b50da7fb59b", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7551677309675142, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:56.066064+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:13:56.066Z" }, { "sequence": 980, "attempt_id": "ad5d332b-3ac9-406c-a3b0-167de7dcbbf0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011384, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.039327295962721, "model_name": "gpt-5.6-sol", "output_tokens": 76, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:13:56.446031+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011384, "created_at": "2026-09-17T03:13:56.446Z" }, { "sequence": 981, "attempt_id": "6d468253-bb8c-4549-8050-1825de939bbb", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.922030708985403, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:13:56.560618+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:13:56.560Z" }, { "sequence": 982, "attempt_id": "0f0d3832-7f24-49ee-add2-0204ecb49ad2", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002346, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.318502958980389, "model_name": "gemini-3.8-flash", "output_tokens": 383, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:13:56.980170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002346, "created_at": "2026-09-17T03:13:56.980Z" }, { "sequence": 983, "attempt_id": "7e881149-935d-4e37-8e01-c80bf636c333", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006192000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.9440258300164714, "model_name": "gpt-5.6-luna", "output_tokens": 105, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:13:57.050096+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006192000000000001, "created_at": "2026-09-17T03:13:57.050Z" }, { "sequence": 984, "attempt_id": "51bf1997-f200-47b2-9c98-ee09bc616bf9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01175875, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 23.651289117056876, "model_name": "muse-spark-1.3", "output_tokens": 2000, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:57.108407+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01175875, "created_at": "2026-09-17T03:13:57.108Z" }, { "sequence": 985, "attempt_id": "00965604-cfb1-43a3-8737-c0534893adf5", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.8700868400046602, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:57.920516+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T03:13:57.920Z" }, { "sequence": 986, "attempt_id": "72f8336a-28a3-4501-a7fe-bc4f4bdac541", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006508, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.1591046950779855, "model_name": "claude-sonnet-5", "output_tokens": 89, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:58.786631+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006508, "created_at": "2026-09-17T03:13:58.786Z" }, { "sequence": 987, "attempt_id": "76727905-bf4b-4f37-9013-c3697ffee86b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.4533480129903182, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:59.441677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005436, "created_at": "2026-09-17T03:13:59.441Z" }, { "sequence": 988, "attempt_id": "c8268d29-55b9-4121-9b28-2ff300db29de", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015895, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 3.1681147660128772, "model_name": "claude-opus-5", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:59.682635+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015895, "created_at": "2026-09-17T03:13:59.682Z" }, { "sequence": 989, "attempt_id": "cf7061b4-30c2-4613-84b4-7c23676210ed", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.821087777032517, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:59.929800+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:13:59.929Z" }, { "sequence": 990, "attempt_id": "8f64e731-5286-4b8b-acf1-ee3b4f096d4a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.0990428420482203, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:00.001367+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004944, "created_at": "2026-09-17T03:14:00.001Z" }, { "sequence": 991, "attempt_id": "a8fa937f-ab34-4b68-a7a5-5c1e6f0842c2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010732, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.825886646984145, "model_name": "gpt-5.6-sol", "output_tokens": 59, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:00.067463+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010732, "created_at": "2026-09-17T03:14:00.067Z" }, { "sequence": 992, "attempt_id": "a81f7578-7d11-483e-8b8b-b8329c6f644e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.026495, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 7.470003409078345, "model_name": "claude-opus-5", "output_tokens": 473, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:00.237181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.026495, "created_at": "2026-09-17T03:14:00.237Z" }, { "sequence": 993, "attempt_id": "3dc58cca-33df-4017-bc2c-e9c5b748d28f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 4.4776264450047165, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:00.610077+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T03:14:00.610Z" }, { "sequence": 994, "attempt_id": "38994839-f3d2-4fc4-a1d6-06488a21bba5", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.804426756920293, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:00.806187+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000368, "created_at": "2026-09-17T03:14:00.806Z" }, { "sequence": 995, "attempt_id": "e2978c1c-698d-4dd5-ab91-13a1b31ba66d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001785, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 2.6107483960222453, "model_name": "gemini-3.8-flash", "output_tokens": 249, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:02.347390+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001785, "created_at": "2026-09-17T03:14:02.347Z" }, { "sequence": 996, "attempt_id": "33be57ff-f5d8-47e9-8c87-f5fb8d9280f5", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01121, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.06896520103328, "model_name": "gemini-3.1-pro-preview", "output_tokens": 732, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:02.511253+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01121, "created_at": "2026-09-17T03:14:02.511Z" }, { "sequence": 997, "attempt_id": "f9c3ac49-3bbc-49f6-b64a-d82100e86b68", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.759688265970908, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:02.633883+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:14:02.634Z" }, { "sequence": 998, "attempt_id": "ad218518-e157-4ec6-aed0-841fa628d9cb", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.0914885730016977, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:03.509727+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:14:03.509Z" }, { "sequence": 999, "attempt_id": "2e415666-94dd-4290-8e03-ca91728e3740", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.227574602002278, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:03.582004+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:14:03.582Z" }, { "sequence": 1000, "attempt_id": "d89936a6-6da9-49ff-890e-c01561642fbb", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 4.177867999998853, "model_name": "gemini-3.1-pro-preview", "output_tokens": 295, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:03.675572+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T03:14:03.675Z" }, { "sequence": 1001, "attempt_id": "0553e7e0-595a-4969-8686-6a4bbcec077a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00570275, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 3.6394311049953103, "model_name": "muse-spark-1.2", "output_tokens": 598, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:03.774713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00570275, "created_at": "2026-09-17T03:14:03.774Z" }, { "sequence": 1002, "attempt_id": "24599603-5e27-4e78-bdd9-d90a0e7a0a49", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 3.4238351460080594, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:04.104328+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:14:04.104Z" }, { "sequence": 1003, "attempt_id": "8d4f60c7-7eed-4891-9708-7b9bd4008707", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.6462663799757138, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:04.348324+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:14:04.348Z" }, { "sequence": 1004, "attempt_id": "8f490830-d325-4c7a-8112-a637cfa057e6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9536194039974362, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:04.535970+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:14:04.536Z" }, { "sequence": 1005, "attempt_id": "2904357f-d382-44e9-9fa0-714954faca36", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.0227839429862797, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:04.609199+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:14:04.609Z" }, { "sequence": 1006, "attempt_id": "e2654aeb-f94b-48a0-aea0-6bc3380d649e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0245625799288973, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:04.677203+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:14:04.677Z" }, { "sequence": 1007, "attempt_id": "d0e42365-c607-4785-89cc-236900832e2e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00488675, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 4.636287408997305, "model_name": "muse-spark-1.3", "output_tokens": 406, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:04.769554+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00488675, "created_at": "2026-09-17T03:14:04.769Z" }, { "sequence": 1008, "attempt_id": "e28f254a-c52b-4abb-9bd0-a188ab4f385b", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.9211576940724626, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:05.096153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:14:05.096Z" }, { "sequence": 1009, "attempt_id": "631c34b0-0429-4474-8382-9ed5e547eb17", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012615, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.7547025040257722, "model_name": "gemini-3.8-flash", "output_tokens": 113, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:05.597081+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0012615, "created_at": "2026-09-17T03:14:05.597Z" }, { "sequence": 1010, "attempt_id": "7a49194b-10f7-4207-96bd-a9fd7185a37a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004163, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.4691303839208558, "model_name": "muse-spark-1.2", "output_tokens": 241, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:06.073624+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004163, "created_at": "2026-09-17T03:14:06.073Z" }, { "sequence": 1011, "attempt_id": "fa4d5115-2faf-4088-82c0-04e6a42365ac", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 1.2942894760053605, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:06.461918+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:14:06.462Z" }, { "sequence": 1012, "attempt_id": "da32ba9d-ea0c-41f9-bbed-57944a9727e1", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.678826026036404, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:06.520148+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T03:14:06.520Z" }, { "sequence": 1013, "attempt_id": "9853a0ae-30cf-4590-8c85-788709eebc85", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004248, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.5761458890046924, "model_name": "muse-spark-1.3", "output_tokens": 261, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:06.981620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004248, "created_at": "2026-09-17T03:14:06.981Z" }, { "sequence": 1014, "attempt_id": "a7a69333-0afc-486c-b475-ecedf3c7d3ab", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.2689813759643584, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:07.052641+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T03:14:07.052Z" }, { "sequence": 1015, "attempt_id": "f81f273f-1ac7-4a14-8fee-35692eaac72f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1415502399904653, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:07.272830+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:14:07.272Z" }, { "sequence": 1016, "attempt_id": "d1b8efcd-9fe8-46d1-a3f0-bf5e212076d2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.0479819759493694, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:07.624626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:14:07.624Z" }, { "sequence": 1017, "attempt_id": "23048935-5fb2-40aa-9818-5d28a6231c2b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.047521526925266, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:07.701331+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:14:07.701Z" }, { "sequence": 1018, "attempt_id": "4d1d090b-5114-4981-8a92-af6b4fd7d74c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003698, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 4.103257143055089, "model_name": "gemini-3.1-pro-preview", "output_tokens": 122, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:07.846389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003698, "created_at": "2026-09-17T03:14:07.846Z" }, { "sequence": 1019, "attempt_id": "77adc890-9021-4626-8256-28543006c1bb", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.3339415109949186, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:08.011452+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:14:08.011Z" }, { "sequence": 1020, "attempt_id": "c3f2ad9d-b188-44b9-be10-e0278b4e9685", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01375625, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 12.197680026991293, "model_name": "muse-spark-1.2", "output_tokens": 2470, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:08.255818+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01375625, "created_at": "2026-09-17T03:14:08.256Z" }, { "sequence": 1021, "attempt_id": "8c36f5fc-9719-472e-9b4d-2b56e8cc20df", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.989675673074089, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:08.318673+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036389999999999995, "created_at": "2026-09-17T03:14:08.318Z" }, { "sequence": 1022, "attempt_id": "dc3d6ec6-2eb5-41a5-8410-0def86773545", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.9663203089730814, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:09.340991+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:14:09.341Z" }, { "sequence": 1023, "attempt_id": "f86a4eac-d84b-451d-a3c7-cff3fde4e9bd", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00227475, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.2944600901100785, "model_name": "gemini-3.8-flash", "output_tokens": 374, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:09.402896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00227475, "created_at": "2026-09-17T03:14:09.402Z" }, { "sequence": 1024, "attempt_id": "de893c91-9516-42f4-b7c0-891b7c3d212a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8085565499495715, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:10.122560+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:14:10.122Z" }, { "sequence": 1025, "attempt_id": "47da524c-25c1-4223-a801-3b620986cae4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007116, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.983606338966638, "model_name": "gpt-5.6-terra", "output_tokens": 190, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:10.504052+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007116, "created_at": "2026-09-17T03:14:10.504Z" }, { "sequence": 1026, "attempt_id": "73c729ec-27a2-4fca-b4e6-cf64074e1d0b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1043783189961687, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:10.564333+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:14:10.564Z" }, { "sequence": 1027, "attempt_id": "95de8ada-716e-48fa-85d7-b15f2c319ea9", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.5571884739911184, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:10.627365+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:14:10.627Z" }, { "sequence": 1028, "attempt_id": "50ca9bad-6a90-43c6-a19f-aace77dc349d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.060978532070294, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:10.963936+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:14:10.964Z" }, { "sequence": 1029, "attempt_id": "5a768bec-0bff-41fa-9dcc-d6cb122d0a29", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7067234340356663, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:11.105581+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:14:11.105Z" }, { "sequence": 1030, "attempt_id": "b984ebdd-f055-440f-8c22-461c6ca15a18", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2081568000139669, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:11.390153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:14:11.390Z" }, { "sequence": 1031, "attempt_id": "7e76c8c2-dce8-4402-9925-a6811440db4b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9388394460547715, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:11.503430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:14:11.503Z" }, { "sequence": 1032, "attempt_id": "b388571b-b8e6-4591-981c-56d6616f9f4c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-014", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00685, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.657066929969005, "model_name": "gemini-3.1-pro-preview", "output_tokens": 377, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:14:11.710001+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00685, "created_at": "2026-09-17T03:14:11.710Z" }, { "sequence": 1033, "attempt_id": "13fcad46-42db-4d89-a8ad-ff29f9a4476a", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.0821924679912627, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:12.105603+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:14:12.105Z" }, { "sequence": 1034, "attempt_id": "a4d5de41-9ec5-4519-947b-23ed227039ef", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00130275, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.719891680055298, "model_name": "gemini-3.8-flash", "output_tokens": 124, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:12.402533+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00130275, "created_at": "2026-09-17T03:14:12.402Z" }, { "sequence": 1035, "attempt_id": "cfdb2010-cbb9-4e6f-8413-6ad34e685d5c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.5655918620759621, "model_name": "muse-spark-1.2", "output_tokens": 197, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:13.014876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003976, "created_at": "2026-09-17T03:14:13.015Z" }, { "sequence": 1036, "attempt_id": "b7d7fd1e-75e2-40be-98d3-575872c33155", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.5659048949601129, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:13.080813+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T03:14:13.081Z" }, { "sequence": 1037, "attempt_id": "89836f07-50dc-41ec-bfb1-85080e546363", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040015, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.113903548917733, "model_name": "muse-spark-1.3", "output_tokens": 203, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:13.277144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0040015, "created_at": "2026-09-17T03:14:13.277Z" }, { "sequence": 1038, "attempt_id": "9d01ecc8-348e-4eb5-ac8f-7f14e87ea3aa", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.792643403983675, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:13.957466+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005758, "created_at": "2026-09-17T03:14:13.957Z" }, { "sequence": 1039, "attempt_id": "15b85051-eacc-462c-90b3-4aae6c6e007c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003494, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.3431399040855467, "model_name": "gemini-3.1-pro-preview", "output_tokens": 105, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:14.023289+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003494, "created_at": "2026-09-17T03:14:14.023Z" }, { "sequence": 1040, "attempt_id": "524bd2e8-0b4f-46c9-a750-a18db3779964", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.4698624680750072, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:14.609639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:14:14.609Z" }, { "sequence": 1041, "attempt_id": "141aa563-c1ee-45cf-a320-1f96794a92d9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00538, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.7078105109976605, "model_name": "gpt-5.6-terra", "output_tokens": 49, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:15.045995+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00538, "created_at": "2026-09-17T03:14:15.046Z" }, { "sequence": 1042, "attempt_id": "4ad21589-ed00-4a90-bc29-8db39018aea5", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.0350079509662464, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:15.116047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:14:15.116Z" }, { "sequence": 1043, "attempt_id": "1da49ba3-635a-4a92-abc8-65962d70ce96", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00925925, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.420080962008797, "model_name": "muse-spark-1.2", "output_tokens": 1426, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:15.176081+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00925925, "created_at": "2026-09-17T03:14:15.176Z" }, { "sequence": 1044, "attempt_id": "f167cf7a-f107-4d9b-bd0e-c141092575a0", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 3.634553916985169, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:15.401546+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014395, "created_at": "2026-09-17T03:14:15.401Z" }, { "sequence": 1045, "attempt_id": "44a65303-b02e-484f-91a5-0279842776c7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.4576550150522962, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:15.481189+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:14:15.481Z" }, { "sequence": 1046, "attempt_id": "7b332cf5-69e2-46a0-8f10-1ae968859ec3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0116525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 19.90948536200449, "model_name": "muse-spark-1.3", "output_tokens": 1975, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:15.651660+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0116525, "created_at": "2026-09-17T03:14:15.651Z" }, { "sequence": 1047, "attempt_id": "2882ca66-1fec-4bca-b2a6-67186e442d7f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.8349175590556115, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:15.940144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003707, "created_at": "2026-09-17T03:14:15.940Z" }, { "sequence": 1048, "attempt_id": "3c45d54a-789f-4c0c-8986-a2a73146f04a", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00143925, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.9860886350506917, "model_name": "gemini-3.8-flash", "output_tokens": 155, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:16.654749+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00143925, "created_at": "2026-09-17T03:14:16.654Z" }, { "sequence": 1049, "attempt_id": "bcda5488-d6ab-4ef0-b4da-b7a53b32c1ad", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-002", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.7154779200209305, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:16.740346+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001884, "created_at": "2026-09-17T03:14:16.740Z" }, { "sequence": 1050, "attempt_id": "10730dd5-006d-48fc-a1f9-7325a111c0b0", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 6.003187703900039, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:17.563848+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T03:14:17.564Z" }, { "sequence": 1051, "attempt_id": "06fa58a6-88c5-4f08-9d8a-1fae77dc224c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00479175, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.5323234780225903, "model_name": "muse-spark-1.2", "output_tokens": 381, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:17.763617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00479175, "created_at": "2026-09-17T03:14:17.763Z" }, { "sequence": 1052, "attempt_id": "21c4b93d-ea3f-41f4-8a66-05e9e2eeed09", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00362, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 3.7591694039292634, "model_name": "gemini-3.1-pro-preview", "output_tokens": 111, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:17.841235+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00362, "created_at": "2026-09-17T03:14:17.844Z" }, { "sequence": 1053, "attempt_id": "732b31d5-1c3f-4d22-a586-8ca71a927862", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.9721842280123383, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:18.699994+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02484, "created_at": "2026-09-17T03:14:18.700Z" }, { "sequence": 1054, "attempt_id": "df9b6191-f61f-44a1-b453-d886361629fc", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01879, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.2400063049281016, "model_name": "claude-opus-5", "output_tokens": 178, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:18.779019+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01879, "created_at": "2026-09-17T03:14:18.779Z" }, { "sequence": 1055, "attempt_id": "c4388dea-b3a6-4755-9627-6343ce67fa93", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.9868499799631536, "model_name": "gpt-5.6-sol", "output_tokens": 70, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:18.850777+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011056, "created_at": "2026-09-17T03:14:18.850Z" }, { "sequence": 1056, "attempt_id": "5855c4b5-b35d-4b08-89c1-944dd0f15f0e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02925, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 3.379492409992963, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:18.922427+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02925, "created_at": "2026-09-17T03:14:18.922Z" }, { "sequence": 1057, "attempt_id": "588cf478-c3fb-4074-9e68-58c662bafba1", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006956, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.1850441429996863, "model_name": "claude-sonnet-5", "output_tokens": 122, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:18.994471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006956, "created_at": "2026-09-17T03:14:18.994Z" }, { "sequence": 1058, "attempt_id": "20b2eaf0-eea0-4319-be0d-d066830989b0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00562, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.7139569050632417, "model_name": "gpt-5.6-terra", "output_tokens": 66, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:19.357519+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00562, "created_at": "2026-09-17T03:14:19.357Z" }, { "sequence": 1059, "attempt_id": "37f5cf98-4cad-434a-a2a4-3839f24485fd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005692000000000001, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6436362860258669, "model_name": "gpt-5.6-luna", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:19.485192+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005692000000000001, "created_at": "2026-09-17T03:14:19.485Z" }, { "sequence": 1060, "attempt_id": "7d7f786d-9f34-45f4-a863-bfacd594ec36", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0045325, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 4.324630446033552, "model_name": "muse-spark-1.3", "output_tokens": 320, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:19.544167+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0045325, "created_at": "2026-09-17T03:14:19.544Z" }, { "sequence": 1061, "attempt_id": "e44c8f68-36e8-4481-84c4-d6276ff60c52", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.9277273330371827, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:19.778832+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003761, "created_at": "2026-09-17T03:14:19.779Z" }, { "sequence": 1062, "attempt_id": "313f54ac-48b3-4d21-8f2a-ae84662cd395", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.5528238900005817, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:20.154539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:14:20.154Z" }, { "sequence": 1063, "attempt_id": "c1b8d982-3ec7-4fdd-bab2-3d2150371c58", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00258525, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 2.113141109002754, "model_name": "gemini-3.8-flash", "output_tokens": 457, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:20.889594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00258525, "created_at": "2026-09-17T03:14:20.889Z" }, { "sequence": 1064, "attempt_id": "76897cc0-4b1a-455a-a74c-20a366a4329a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.7562642869306728, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:21.300758+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:14:21.300Z" }, { "sequence": 1065, "attempt_id": "dea8341a-e021-4e65-a7e6-a67ab0737ff2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.120809015003033, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:21.362939+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:14:21.364Z" }, { "sequence": 1066, "attempt_id": "66c39474-f3fa-4023-b860-6988a34058d9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0090085, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 13.689074197085574, "model_name": "muse-spark-1.3", "output_tokens": 1367, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:21.438165+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0090085, "created_at": "2026-09-17T03:14:21.438Z" }, { "sequence": 1067, "attempt_id": "da19da58-56dd-4190-8a96-c52cbbbe543b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.9839594309451059, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:21.499364+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:14:21.499Z" }, { "sequence": 1068, "attempt_id": "bd36422b-a8af-4562-a8b6-debb5325a7b4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0580253720982, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:22.010408+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:14:22.010Z" }, { "sequence": 1069, "attempt_id": "ccab72df-4a7d-4274-b9c1-41e6938c5e36", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.3254287829622626, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:22.175709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:14:22.175Z" }, { "sequence": 1070, "attempt_id": "c7cd100c-41ed-415a-bdd5-837528ae61bd", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.5249353679828346, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:22.584448+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:14:22.584Z" }, { "sequence": 1071, "attempt_id": "46a85065-0984-4621-a23e-d5316f6ca9fa", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00138525, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.4185776710510254, "model_name": "gemini-3.8-flash", "output_tokens": 142, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:22.977933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00138525, "created_at": "2026-09-17T03:14:22.978Z" }, { "sequence": 1072, "attempt_id": "8207bb8c-0756-4b86-ac84-83a5154a86d8", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.6507097979774699, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:23.212294+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:14:23.212Z" }, { "sequence": 1073, "attempt_id": "56dcdae8-94e2-4f45-bc3c-5f4f06768874", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.9467823910526931, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:23.310072+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:14:23.310Z" }, { "sequence": 1074, "attempt_id": "f4c1801e-0966-40af-9099-6f8a328dc7da", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.8655119260074571, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:24.235550+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:14:24.235Z" }, { "sequence": 1075, "attempt_id": "7509c437-39df-468b-b207-37902493e5d9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00457475, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.379415131988935, "model_name": "muse-spark-1.2", "output_tokens": 332, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:24.616131+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00457475, "created_at": "2026-09-17T03:14:24.616Z" }, { "sequence": 1076, "attempt_id": "2d4581dc-cc62-4b57-a66e-ddbb196e4737", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.027457650983706, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:25.065717+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:14:25.065Z" }, { "sequence": 1077, "attempt_id": "4f349d79-9730-4c86-a32e-f46c3805064c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8525107860332355, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:25.128153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:14:25.128Z" }, { "sequence": 1078, "attempt_id": "4217ba3d-72bd-4ed2-be0b-8e4db8676b32", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004926, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.059048336930573, "model_name": "gemini-3.1-pro-preview", "output_tokens": 221, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:25.497566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004926, "created_at": "2026-09-17T03:14:25.497Z" }, { "sequence": 1079, "attempt_id": "0927771d-de93-4be8-91ee-e2248f9ba085", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004681, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.60226803610567, "model_name": "muse-spark-1.3", "output_tokens": 357, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:25.672990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004681, "created_at": "2026-09-17T03:14:25.673Z" }, { "sequence": 1080, "attempt_id": "d9badc0e-1292-4051-a76c-38046dd7b451", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00839275, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 6.8556014619534835, "model_name": "muse-spark-1.2", "output_tokens": 1223, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:25.850377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00839275, "created_at": "2026-09-17T03:14:25.850Z" }, { "sequence": 1081, "attempt_id": "ea00a540-9945-46ae-a67f-3a55fb0df873", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7065674240002409, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:26.004279+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:14:26.004Z" }, { "sequence": 1082, "attempt_id": "543f47d8-f083-4744-b5bd-99044211351c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9286804689327255, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:26.117902+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:14:26.118Z" }, { "sequence": 1083, "attempt_id": "82480d72-7b2e-41b4-9771-05b86b94ce4e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1868378699291497, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:26.315284+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:14:26.315Z" }, { "sequence": 1084, "attempt_id": "fc3c3997-6237-480f-9d06-17d4d231225b", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.2616434829542413, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:27.172369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00035009999999999994, "created_at": "2026-09-17T03:14:27.172Z" }, { "sequence": 1085, "attempt_id": "c60cef45-e089-4283-8bd1-ff901c4e4b90", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.61533987603616, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:27.260873+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:14:27.260Z" }, { "sequence": 1086, "attempt_id": "f8d6447b-29c5-4ec8-b128-5323b4c61f7f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129525, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.6078935300465673, "model_name": "gemini-3.8-flash", "output_tokens": 122, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:27.340420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00129525, "created_at": "2026-09-17T03:14:27.340Z" }, { "sequence": 1087, "attempt_id": "e7eb7647-56af-4c3f-8524-0d7607a77f21", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00419275, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.84347283502575, "model_name": "muse-spark-1.2", "output_tokens": 248, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:28.021292+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00419275, "created_at": "2026-09-17T03:14:28.021Z" }, { "sequence": 1088, "attempt_id": "1592fc0d-a204-4eb0-89eb-c7443d82b82f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6487378149759024, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:28.083656+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T03:14:28.083Z" }, { "sequence": 1089, "attempt_id": "c04dda89-f276-4e8f-a4e5-8f424782d2ae", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.885169890942052, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:28.562807+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:14:28.562Z" }, { "sequence": 1090, "attempt_id": "cd1b43f1-76d0-4b29-a2ae-c40fe798c497", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00451575, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.6159100950462744, "model_name": "muse-spark-1.3", "output_tokens": 324, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:28.680567+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00451575, "created_at": "2026-09-17T03:14:28.680Z" }, { "sequence": 1091, "attempt_id": "2056c8a5-901f-49cf-a4f5-33377ea9fafc", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.7612474620109424, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:29.080278+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T03:14:29.080Z" }, { "sequence": 1092, "attempt_id": "8a642eec-e4bb-4074-8868-f607f211609c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.845548804034479, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:29.141869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014315, "created_at": "2026-09-17T03:14:29.143Z" }, { "sequence": 1093, "attempt_id": "c1989666-9466-4beb-ad99-ced94fd20422", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3193453359417617, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:29.461485+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:14:29.461Z" }, { "sequence": 1094, "attempt_id": "eefc0821-1350-4c9f-b8d3-b5f4af1cf381", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.346115943044424, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:29.720760+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T03:14:29.720Z" }, { "sequence": 1095, "attempt_id": "3367162e-741b-4627-8906-bb92560f4baa", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6564099489478394, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:29.782148+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:14:29.782Z" }, { "sequence": 1096, "attempt_id": "638ba0e7-b3b5-4f5d-9f99-00087087b289", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005332, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3936986789340153, "model_name": "gpt-5.6-terra", "output_tokens": 46, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:30.017903+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005332, "created_at": "2026-09-17T03:14:30.018Z" }, { "sequence": 1097, "attempt_id": "1a07d3ca-e02b-4200-87ae-cf02926d64d9", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.019472, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 12.169177787029184, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1429, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:30.080047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.019472, "created_at": "2026-09-17T03:14:30.080Z" }, { "sequence": 1098, "attempt_id": "0ef1ceb7-02c7-4e9b-b0e3-cbc936778051", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3534253040561453, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:30.139810+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:14:30.140Z" }, { "sequence": 1099, "attempt_id": "62bae608-f073-424e-9e87-06d9da46eaf8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006816, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 11.227890470996499, "model_name": "muse-spark-1.3", "output_tokens": 852, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:30.219711+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006816, "created_at": "2026-09-17T03:14:30.219Z" }, { "sequence": 1100, "attempt_id": "fcec1934-13e3-4aef-86e0-36bb05b6ff2f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.9647500870050862, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:30.486343+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:14:30.486Z" }, { "sequence": 1101, "attempt_id": "ef4410b4-297e-43f2-8e88-9d7bcc770124", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.6339506580261514, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:30.912076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001833, "created_at": "2026-09-17T03:14:30.912Z" }, { "sequence": 1102, "attempt_id": "99014cb3-7f2e-4b7a-8de2-4b95c73ebba9", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00180525, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.8732471339171752, "model_name": "gemini-3.8-flash", "output_tokens": 254, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:31.074194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00180525, "created_at": "2026-09-17T03:14:31.074Z" }, { "sequence": 1103, "attempt_id": "abdea9ac-7c48-4b1d-ab7d-9d2c1436cf50", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00407325, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.4419071458978578, "model_name": "muse-spark-1.2", "output_tokens": 214, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:31.282950+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00407325, "created_at": "2026-09-17T03:14:31.283Z" }, { "sequence": 1104, "attempt_id": "621bcaf5-f46a-43b2-a33b-c5ae73c9333d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008078, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 6.038641278981231, "model_name": "gemini-3.1-pro-preview", "output_tokens": 487, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:31.596639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008078, "created_at": "2026-09-17T03:14:31.596Z" }, { "sequence": 1105, "attempt_id": "7a79b999-e82d-440c-8e20-b57762fb21a4", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.7446646600728855, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:31.944286+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005684, "created_at": "2026-09-17T03:14:31.944Z" }, { "sequence": 1106, "attempt_id": "843b0279-2cbf-4f98-86d7-ae6037c5d4bd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.7371187779353932, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:32.091852+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004904, "created_at": "2026-09-17T03:14:32.091Z" }, { "sequence": 1107, "attempt_id": "e5d23baa-24fd-4482-83aa-8bcb6e76a568", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 2.137903729919344, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:32.278027+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01421, "created_at": "2026-09-17T03:14:32.278Z" }, { "sequence": 1108, "attempt_id": "adc644fd-1562-498b-8e69-510eae7f3e44", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.1652702399296686, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:32.344876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004904, "created_at": "2026-09-17T03:14:32.346Z" }, { "sequence": 1109, "attempt_id": "603a82c2-8d7d-4ab2-a342-d93bbf7e05d2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.7694938209606335, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:32.743299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009752, "created_at": "2026-09-17T03:14:32.743Z" }, { "sequence": 1110, "attempt_id": "0e7d123a-8614-42cd-b6a9-993dfcfb6973", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.4805010419804603, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:33.028541+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02438, "created_at": "2026-09-17T03:14:33.028Z" }, { "sequence": 1111, "attempt_id": "98bb299e-d414-4157-8584-4863bf4e888c", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00112875, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3733802980277687, "model_name": "gemini-3.8-flash", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:33.379230+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00112875, "created_at": "2026-09-17T03:14:33.379Z" }, { "sequence": 1112, "attempt_id": "2e2ee9b6-4abc-4153-ac7f-3f7a544fd048", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00467675, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.6972280289046466, "model_name": "muse-spark-1.3", "output_tokens": 356, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:33.479650+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00467675, "created_at": "2026-09-17T03:14:33.479Z" }, { "sequence": 1113, "attempt_id": "e141118d-2bb2-4b0e-b51d-85d9d6f4fd96", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003914, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.0989621100015938, "model_name": "muse-spark-1.2", "output_tokens": 183, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:33.538382+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003914, "created_at": "2026-09-17T03:14:33.539Z" }, { "sequence": 1114, "attempt_id": "08048c98-707e-4543-9851-55bf89cfef8d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.40542722993996, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:33.597882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:14:33.599Z" }, { "sequence": 1115, "attempt_id": "b8d286c8-2f2b-4f65-b68f-20df77f3f4ff", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 3.5332975139608607, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:33.657755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02844, "created_at": "2026-09-17T03:14:33.657Z" }, { "sequence": 1116, "attempt_id": "a128121a-739a-47f9-b35f-85f77aa9551c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00374825, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.392521302914247, "model_name": "muse-spark-1.3", "output_tokens": 144, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:33.737673+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00374825, "created_at": "2026-09-17T03:14:33.737Z" }, { "sequence": 1117, "attempt_id": "f9a907d5-36ce-4e30-b568-fe6d3ca61533", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00645, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.135882672038861, "model_name": "gemini-3.1-pro-preview", "output_tokens": 348, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:34.278056+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00645, "created_at": "2026-09-17T03:14:34.278Z" }, { "sequence": 1118, "attempt_id": "0d0f4169-5b91-44bc-90c1-ac6ecb095ecf", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.7597760929493234, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:34.340151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T03:14:34.341Z" }, { "sequence": 1119, "attempt_id": "9001ba8e-8181-4460-95a3-bd6a1cffe027", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01432, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 1.8323320890776813, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:34.921104+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01432, "created_at": "2026-09-17T03:14:34.921Z" }, { "sequence": 1120, "attempt_id": "548f4c81-ce86-4d97-81e9-c7e093502edd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005448, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.3175878380425274, "model_name": "gpt-5.6-luna", "output_tokens": 56, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:35.113566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005448, "created_at": "2026-09-17T03:14:35.113Z" }, { "sequence": 1121, "attempt_id": "f9056edc-ff41-4f64-afbc-f8fa6d3750b6", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003754, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 3.6524202859727666, "model_name": "gemini-3.1-pro-preview", "output_tokens": 127, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:35.311368+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003754, "created_at": "2026-09-17T03:14:35.311Z" }, { "sequence": 1122, "attempt_id": "003dcca1-4086-4256-bb7e-48970b0c3b91", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005412, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.615151818958111, "model_name": "gpt-5.6-terra", "output_tokens": 53, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:35.373049+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005412, "created_at": "2026-09-17T03:14:35.374Z" }, { "sequence": 1123, "attempt_id": "62826dc2-4826-40a4-8866-0eb5ed02f06b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.8833877890137956, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:35.481521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:14:35.481Z" }, { "sequence": 1124, "attempt_id": "6786cbe8-f543-4551-a681-9fb855a08a28", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010932, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.935380549984984, "model_name": "gpt-5.6-sol", "output_tokens": 69, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:35.593411+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010932, "created_at": "2026-09-17T03:14:35.593Z" }, { "sequence": 1125, "attempt_id": "45a838c1-f760-472c-a76d-a432be53acd5", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006478, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.2406098790233955, "model_name": "claude-sonnet-5", "output_tokens": 86, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:35.681107+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006478, "created_at": "2026-09-17T03:14:35.681Z" }, { "sequence": 1126, "attempt_id": "1f2ed844-521e-4b6f-93ac-cc4e5a938fee", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00175875, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.9074553101090714, "model_name": "gemini-3.8-flash", "output_tokens": 242, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:36.307410+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00175875, "created_at": "2026-09-17T03:14:36.307Z" }, { "sequence": 1127, "attempt_id": "85be7a31-b3ae-4048-8d35-945ba2367c26", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.4478617920540273, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:36.432102+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000368, "created_at": "2026-09-17T03:14:36.432Z" }, { "sequence": 1128, "attempt_id": "5cbb7df0-7561-456e-ab75-40300124f263", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.7774441180517897, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:36.519461+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:14:36.519Z" }, { "sequence": 1129, "attempt_id": "6944f9c0-14bc-4041-82d2-c4adff358b3f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.6920058819232509, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:37.348761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T03:14:37.348Z" }, { "sequence": 1130, "attempt_id": "5b76cd85-89ff-4b71-83fc-239bd58fc317", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004823, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 2.401786943897605, "model_name": "muse-spark-1.2", "output_tokens": 391, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:37.775223+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004823, "created_at": "2026-09-17T03:14:37.775Z" }, { "sequence": 1131, "attempt_id": "92685a2e-611a-44a9-b7eb-745a0470d3c1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010472, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4885022500529885, "model_name": "gpt-5.6-sol", "output_tokens": 40, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:37.981794+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010472, "created_at": "2026-09-17T03:14:37.981Z" }, { "sequence": 1132, "attempt_id": "38cb4b11-e7fe-4b57-9e4b-a45988eb4d85", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.6783430599607527, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:38.052596+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:14:38.052Z" }, { "sequence": 1133, "attempt_id": "85dce5bd-7093-4ad8-935f-7ba48ded822b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.577036325004883, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:38.123432+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T03:14:38.123Z" }, { "sequence": 1134, "attempt_id": "84dfbde1-c526-475f-8388-77528dc0e82a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1466336160665378, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:38.559200+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:14:38.559Z" }, { "sequence": 1135, "attempt_id": "82436c08-b8f2-4e77-98fb-24c394e5750b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.3775680540129542, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:38.821609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:14:38.821Z" }, { "sequence": 1136, "attempt_id": "fcd7c131-e656-4485-b937-5d2ae396e662", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.1145819310331717, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:39.238368+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003764, "created_at": "2026-09-17T03:14:39.238Z" }, { "sequence": 1137, "attempt_id": "da10e6b1-0bc9-4849-8c9d-29d7cbfa6c52", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049335, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 4.074470401974395, "model_name": "muse-spark-1.3", "output_tokens": 417, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:39.305380+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049335, "created_at": "2026-09-17T03:14:39.305Z" }, { "sequence": 1138, "attempt_id": "5b8de10e-3d98-4ce8-9f48-827739f2f63d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005532, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.7908517440082505, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:39.376526+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005532, "created_at": "2026-09-17T03:14:39.376Z" }, { "sequence": 1139, "attempt_id": "ab06423b-3637-4b82-9979-6aefa584edf4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008042, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.349751467001624, "model_name": "gemini-3.1-pro-preview", "output_tokens": 481, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:39.690245+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008042, "created_at": "2026-09-17T03:14:39.690Z" }, { "sequence": 1140, "attempt_id": "8c25f603-82f8-4acc-a5e5-84da1a47df5f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.9393656679894775, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:40.380993+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T03:14:40.381Z" }, { "sequence": 1141, "attempt_id": "09e1123f-bb09-461d-ac82-de4fd239698a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 7.876769835012965, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:40.681877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T03:14:40.682Z" }, { "sequence": 1142, "attempt_id": "4f8b7c9d-84db-4e8d-95c3-d41ae11ec720", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003786, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.9755336169619113, "model_name": "gemini-3.8-flash", "output_tokens": 777, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:41.028606+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003786, "created_at": "2026-09-17T03:14:41.028Z" }, { "sequence": 1143, "attempt_id": "7208ebed-349d-42c9-9d9d-e07246e232a2", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.7538605600129813, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:41.512129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:14:41.512Z" }, { "sequence": 1144, "attempt_id": "ed87c357-e2d9-43cb-9263-64134c00e0e5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.2054755940334871, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:41.669574+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:14:41.669Z" }, { "sequence": 1145, "attempt_id": "68a385d4-2d7d-41dd-aee0-3d02e64bd88c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006708, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.7715131839504465, "model_name": "claude-sonnet-5", "output_tokens": 108, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:42.143966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006708, "created_at": "2026-09-17T03:14:42.144Z" }, { "sequence": 1146, "attempt_id": "4519996f-379c-4fa0-81fc-d27872cc3e31", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016345, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.9129479459952563, "model_name": "claude-opus-5", "output_tokens": 91, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:42.218646+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016345, "created_at": "2026-09-17T03:14:42.218Z" }, { "sequence": 1147, "attempt_id": "5197b3b9-72a2-40c1-8cd8-6b59e1f4665c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005426, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.618629898992367, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:42.364757+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005426, "created_at": "2026-09-17T03:14:42.364Z" }, { "sequence": 1148, "attempt_id": "8c4368e3-cd72-4ed9-8487-7b2b4eec3764", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.5782261370914057, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:42.511620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T03:14:42.511Z" }, { "sequence": 1149, "attempt_id": "e1e983cd-d27b-4f8c-85d8-e01a6b657946", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.4185114010469988, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:42.576603+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:14:42.576Z" }, { "sequence": 1150, "attempt_id": "a0fdf564-ef82-4532-93d5-5927acab911f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035580000000000003, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.0131515270331874, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:43.223837+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00035580000000000003, "created_at": "2026-09-17T03:14:43.224Z" }, { "sequence": 1151, "attempt_id": "c33c0559-22cb-4bab-ae6d-731f606f2a1e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002142, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.230731250019744, "model_name": "gemini-3.8-flash", "output_tokens": 344, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:43.966227+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002142, "created_at": "2026-09-17T03:14:43.966Z" }, { "sequence": 1152, "attempt_id": "6d0c2bcb-1287-4283-b031-3bb836babbb6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6799873120617121, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:44.710854+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:14:44.711Z" }, { "sequence": 1153, "attempt_id": "b1629444-9dbd-42db-816c-9674ed380323", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016145, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.901952369022183, "model_name": "claude-opus-5", "output_tokens": 78, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:45.541825+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016145, "created_at": "2026-09-17T03:14:45.542Z" }, { "sequence": 1154, "attempt_id": "25d2be43-83cb-4b85-8e1e-ad014f92b097", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00532575, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.3866187180392444, "model_name": "muse-spark-1.2", "output_tokens": 509, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:45.818377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00532575, "created_at": "2026-09-17T03:14:45.818Z" }, { "sequence": 1155, "attempt_id": "bbc109bd-29da-4ec3-b487-389bb5a7e057", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007168, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.8970667470712215, "model_name": "claude-sonnet-5", "output_tokens": 149, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:46.186625+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007168, "created_at": "2026-09-17T03:14:46.186Z" }, { "sequence": 1156, "attempt_id": "9ae5c4df-40f9-4a7a-91d4-f1db4b490532", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009918, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.615564880077727, "model_name": "muse-spark-1.2", "output_tokens": 1581, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:46.289096+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009918, "created_at": "2026-09-17T03:14:46.289Z" }, { "sequence": 1157, "attempt_id": "cb2b8ec3-1ef9-4d2d-aee6-0fcffdb8919e", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006616, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.772019983967766, "model_name": "gemini-3.1-pro-preview", "output_tokens": 362, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:46.355260+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006616, "created_at": "2026-09-17T03:14:46.355Z" }, { "sequence": 1158, "attempt_id": "85588c43-c30e-42df-a5f0-08f9971b7e47", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.8533059089677408, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:46.629851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:14:46.630Z" }, { "sequence": 1159, "attempt_id": "34799a92-0eb9-46f1-8f01-0fa923fc7a2d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.2779374970123172, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:46.884889+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:14:46.885Z" }, { "sequence": 1160, "attempt_id": "7199c288-b844-4a9b-a2de-bb7db95dceb0", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01351, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 9.234951923950575, "model_name": "gemini-3.1-pro-preview", "output_tokens": 932, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:47.104808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01351, "created_at": "2026-09-17T03:14:47.104Z" }, { "sequence": 1161, "attempt_id": "7a7aa253-9c6d-45be-a322-a03d9ecec59a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005532, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.5869403240503743, "model_name": "gpt-5.6-terra", "output_tokens": 61, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:47.470143+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005532, "created_at": "2026-09-17T03:14:47.470Z" }, { "sequence": 1162, "attempt_id": "3b8ed48c-48fa-46d1-aa71-141e46cbacda", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9908346260199323, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:47.685370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:14:47.685Z" }, { "sequence": 1163, "attempt_id": "2a3d21be-8cee-417e-aeb3-8965033c0115", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005856, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7435177769511938, "model_name": "gpt-5.6-luna", "output_tokens": 88, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:47.995402+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005856, "created_at": "2026-09-17T03:14:47.995Z" }, { "sequence": 1164, "attempt_id": "3c4a7797-780a-4426-a54e-3203e5eb3c2f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 5.814442589995451, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:48.391376+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:14:48.391Z" }, { "sequence": 1165, "attempt_id": "98a70ca2-9d7c-484e-8dba-b3512e1b52ef", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6902334289625287, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:49.146740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T03:14:49.146Z" }, { "sequence": 1166, "attempt_id": "44cf8bc1-8dbd-4af0-8b83-e8c8a56cc2d6", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00258375, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.9131213759537786, "model_name": "gemini-3.8-flash", "output_tokens": 459, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:49.334578+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00258375, "created_at": "2026-09-17T03:14:49.334Z" }, { "sequence": 1167, "attempt_id": "f487d2c8-22a5-4d6f-b6eb-13380dcafb05", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.7264767379965633, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:49.787147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:14:49.787Z" }, { "sequence": 1168, "attempt_id": "a9e26633-d5f1-4b53-bbb1-76cf18dbeeac", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.060147244017571, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:49.853454+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T03:14:49.853Z" }, { "sequence": 1169, "attempt_id": "43eebf87-f093-4655-bf18-d0a490d65c47", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004388, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.6385376329999417, "model_name": "gemini-3.1-pro-preview", "output_tokens": 174, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:49.994111+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004388, "created_at": "2026-09-17T03:14:49.994Z" }, { "sequence": 1170, "attempt_id": "fb820920-3c68-41b1-8476-98fc900e111d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005552, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 2.8945549589116126, "model_name": "muse-spark-1.2", "output_tokens": 559, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:50.063719+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005552, "created_at": "2026-09-17T03:14:50.063Z" }, { "sequence": 1171, "attempt_id": "9c3e6527-0c08-42d2-8bbb-13cff2683798", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0933832079172134, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:50.492360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:14:50.492Z" }, { "sequence": 1172, "attempt_id": "a5e2e1db-2001-443e-8e19-91f44d006659", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006792, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 8.40305144910235, "model_name": "muse-spark-1.3", "output_tokens": 854, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:50.685901+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006792, "created_at": "2026-09-17T03:14:50.686Z" }, { "sequence": 1173, "attempt_id": "1fd414dc-57cf-464e-8155-084d5d9dfd9b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.99956258200109, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:50.916357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:14:50.916Z" }, { "sequence": 1174, "attempt_id": "4ccea91a-2e32-4642-91c7-5eca76f00f5e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.70649367000442, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:50.979860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:14:50.980Z" }, { "sequence": 1175, "attempt_id": "6f864808-6574-4193-b5f6-80e0c51efda5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.3068594760261476, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:51.160625+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:14:51.160Z" }, { "sequence": 1176, "attempt_id": "3e71d711-7ea9-46ed-8b47-bde3e01bfc2f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.650585123919882, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:51.225517+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T03:14:51.225Z" }, { "sequence": 1177, "attempt_id": "11fdb403-6858-473c-acb1-e3712c6f5e88", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.8775847860379145, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:51.434669+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:14:51.434Z" }, { "sequence": 1178, "attempt_id": "4d455ced-6fbd-4fa2-93f1-c065698ae792", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.4864134290255606, "model_name": "gemini-3.8-flash", "output_tokens": 105, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:51.612298+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00122925, "created_at": "2026-09-17T03:14:51.612Z" }, { "sequence": 1179, "attempt_id": "f7cbcdf0-2e3e-4851-b8a0-a02f62609fa2", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.9060772069497034, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:52.407512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:14:52.407Z" }, { "sequence": 1180, "attempt_id": "504d3770-3572-4124-8436-cd64d6d3695d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00586225, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.864560827962123, "model_name": "muse-spark-1.3", "output_tokens": 632, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:52.824508+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00586225, "created_at": "2026-09-17T03:14:52.824Z" }, { "sequence": 1181, "attempt_id": "7b3107fe-13bb-424e-a2d9-5aa846dc6980", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.8352821399457753, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:53.124213+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:14:53.124Z" }, { "sequence": 1182, "attempt_id": "2cde75f6-42a9-4766-bf1a-7d8580aa9e48", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004478, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 2.212465983000584, "model_name": "muse-spark-1.2", "output_tokens": 316, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:53.194934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004478, "created_at": "2026-09-17T03:14:53.195Z" }, { "sequence": 1183, "attempt_id": "d84dd688-d9e7-4d85-8d55-f39523736513", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.059077264042571, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:53.284925+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:14:53.285Z" }, { "sequence": 1184, "attempt_id": "7e9e9731-e533-4996-b84c-151898a66350", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6589966209139675, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:53.354077+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:14:53.354Z" }, { "sequence": 1185, "attempt_id": "9731079b-a508-44cf-85d8-d37efa8cc9b2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004325, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.0686949479859322, "model_name": "muse-spark-1.3", "output_tokens": 280, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:53.818412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004325, "created_at": "2026-09-17T03:14:53.818Z" }, { "sequence": 1186, "attempt_id": "6cc1c5ff-338f-4ed9-ae57-5ceb17ca5e54", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.047896070056595, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:53.975071+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:14:53.975Z" }, { "sequence": 1187, "attempt_id": "704a109f-6e53-457f-9c45-52fc19c90d96", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00518, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.9562194580212235, "model_name": "gemini-3.1-pro-preview", "output_tokens": 246, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:54.042238+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00518, "created_at": "2026-09-17T03:14:54.043Z" }, { "sequence": 1188, "attempt_id": "d88411ce-85ea-4a54-a604-885d33b13315", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.858407773077488, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:54.108389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:14:54.108Z" }, { "sequence": 1189, "attempt_id": "da435098-3a9e-4b54-9e17-466932a260b1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01121425, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 16.180186784942634, "model_name": "muse-spark-1.3", "output_tokens": 1886, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:54.375193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01121425, "created_at": "2026-09-17T03:14:54.375Z" }, { "sequence": 1190, "attempt_id": "64dc6f92-f7fc-412b-bd7a-0c8027a9aa86", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.640097302966751, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:54.681847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:14:54.681Z" }, { "sequence": 1191, "attempt_id": "817cb3a4-25fb-4e4d-a00f-97fd07001f3f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.328264475101605, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:54.764126+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003561, "created_at": "2026-09-17T03:14:54.764Z" }, { "sequence": 1192, "attempt_id": "b8a1d2d4-bd19-4193-8a28-0d513afa8622", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.981469350052066, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:55.455695+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:14:55.455Z" }, { "sequence": 1193, "attempt_id": "6eb3ee54-d050-4d14-aef3-1b6a4c0d78d4", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.7701321670319885, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:55.521989+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T03:14:55.522Z" }, { "sequence": 1194, "attempt_id": "b06aa486-ffb5-4fa2-927f-b15237c3d769", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00439625, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.4987557029817253, "model_name": "muse-spark-1.2", "output_tokens": 290, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:55.587887+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00439625, "created_at": "2026-09-17T03:14:55.588Z" }, { "sequence": 1195, "attempt_id": "1840c76a-8eaf-464d-ae87-dbd62d8507ec", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.7306144459871575, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:56.173616+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00601, "created_at": "2026-09-17T03:14:56.173Z" }, { "sequence": 1196, "attempt_id": "2c88dead-6781-4c63-8e32-43c67c2e721e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.7590133970370516, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:56.591269+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:14:56.591Z" }, { "sequence": 1197, "attempt_id": "4c03deef-ae35-4f14-86de-28cefc399ca5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1737439460121095, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:56.696051+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:14:56.696Z" }, { "sequence": 1198, "attempt_id": "2d45a49d-5a7a-4486-8115-9d41ceea0a7d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005442, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1845170219894499, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:56.775600+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005442, "created_at": "2026-09-17T03:14:56.775Z" }, { "sequence": 1199, "attempt_id": "188e31cf-eda2-4b6e-bed5-172ea21cb6ca", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0013515, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.4884608319262043, "model_name": "gemini-3.8-flash", "output_tokens": 133, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:56.845691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0013515, "created_at": "2026-09-17T03:14:56.846Z" }, { "sequence": 1200, "attempt_id": "cd26c8da-6aff-45fb-908c-15657b7290bb", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.6283880199771374, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:56.910944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015025, "created_at": "2026-09-17T03:14:56.911Z" }, { "sequence": 1201, "attempt_id": "99eeca5c-32a4-4187-9407-603d26938b62", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003726, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.6575521510094404, "model_name": "gemini-3.1-pro-preview", "output_tokens": 121, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:56.979411+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003726, "created_at": "2026-09-17T03:14:56.979Z" }, { "sequence": 1202, "attempt_id": "27e1d6fe-4e1c-4a87-906c-9ca463a5eedc", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005718, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.507989592035301, "model_name": "gpt-5.6-luna", "output_tokens": 66, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:57.160303+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005718, "created_at": "2026-09-17T03:14:57.160Z" }, { "sequence": 1203, "attempt_id": "bac10ee5-348c-4608-ac5d-2d78f2a7a38d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00431975, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.332331531913951, "model_name": "muse-spark-1.3", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:57.258446+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00431975, "created_at": "2026-09-17T03:14:57.258Z" }, { "sequence": 1204, "attempt_id": "df6f954a-d675-4dd5-a59d-a38d3b5f637f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.7467956970212981, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:14:57.522704+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:14:57.522Z" }, { "sequence": 1205, "attempt_id": "3c0cb641-45e7-4dda-8fa4-d6fabec946b1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.6810645760269836, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:58.003539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T03:14:58.003Z" }, { "sequence": 1206, "attempt_id": "bcdc78f0-54de-4adb-96c5-fb1d85fbe653", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0017775, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.649504448985681, "model_name": "gemini-3.8-flash", "output_tokens": 232, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:14:58.307171+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0017775, "created_at": "2026-09-17T03:14:58.307Z" }, { "sequence": 1207, "attempt_id": "664df66d-1432-48e9-a323-00069b591a0e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00466725, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 1.7913931099465117, "model_name": "muse-spark-1.2", "output_tokens": 332, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:58.768975+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00466725, "created_at": "2026-09-17T03:14:58.769Z" }, { "sequence": 1208, "attempt_id": "226d4975-74fa-4e94-8d78-e081c129a1e5", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.9959734079893678, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:59.221900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T03:14:59.222Z" }, { "sequence": 1209, "attempt_id": "27d1d0fe-d2b1-4a93-8657-69b1bb87c494", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 0.9121580169303343, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:59.289715+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:14:59.289Z" }, { "sequence": 1210, "attempt_id": "b4dfce1e-4199-4bec-a085-4fbe871b1652", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.436294966028072, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:59.506808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:14:59.506Z" }, { "sequence": 1211, "attempt_id": "c8c0f2d5-cbb4-4b78-bfe9-4f39c719e743", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.8301932819886133, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:59.876965+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T03:14:59.877Z" }, { "sequence": 1212, "attempt_id": "911176ea-2c29-498d-81ab-fba1c897c047", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 5.931666451971978, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:15:00.040375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:15:00.040Z" }, { "sequence": 1213, "attempt_id": "cf48b667-8d12-403f-9508-e55ef2614684", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005408, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.104723145952448, "model_name": "gemini-3.1-pro-preview", "output_tokens": 249, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:15:00.344383+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005408, "created_at": "2026-09-17T03:15:00.344Z" }, { "sequence": 1214, "attempt_id": "969d3dfb-e544-4fcf-9f66-08c9553d4a08", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.0303257709601894, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:00.865977+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:15:00.866Z" }, { "sequence": 1215, "attempt_id": "9a08e8aa-c23c-4668-ba38-6a1670bc45de", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 3.337378764990717, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:00.933371+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:15:00.934Z" }, { "sequence": 1216, "attempt_id": "00ba07c0-c4d0-4123-89b0-6be47ac3f2ec", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0036395, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 1.0562436049804091, "model_name": "muse-spark-1.2", "output_tokens": 119, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:01.162349+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0036395, "created_at": "2026-09-17T03:15:01.162Z" }, { "sequence": 1217, "attempt_id": "4e65020c-aaac-400f-b360-df9350b3581a", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.1774505189387128, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:01.749691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:15:01.749Z" }, { "sequence": 1218, "attempt_id": "919cb3eb-5125-44d8-ac1e-59ca421c60db", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.2248770860023797, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:02.451957+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:15:02.452Z" }, { "sequence": 1219, "attempt_id": "a5f0dafd-8160-4961-9e80-85d7d91df470", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00544925, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 5.6755881229182705, "model_name": "muse-spark-1.3", "output_tokens": 516, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:15:02.524923+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00544925, "created_at": "2026-09-17T03:15:02.526Z" }, { "sequence": 1220, "attempt_id": "405a95fe-c57f-4563-938f-8a8514b59b86", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00417925, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.9343231569509953, "model_name": "muse-spark-1.3", "output_tokens": 246, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:02.877684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00417925, "created_at": "2026-09-17T03:15:02.877Z" }, { "sequence": 1221, "attempt_id": "6c573dba-ef43-4f96-ab10-fc5ea8639616", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00126, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 3.746939786011353, "model_name": "gemini-3.8-flash", "output_tokens": 113, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:03.101791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00126, "created_at": "2026-09-17T03:15:03.101Z" }, { "sequence": 1222, "attempt_id": "c5926544-f74c-41c9-a5e1-a8eacef96cab", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.278180051012896, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:03.211800+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:15:03.211Z" }, { "sequence": 1223, "attempt_id": "dcc72430-95eb-41bb-959c-ceec550d18c0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0487281819805503, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:03.574000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:15:03.574Z" }, { "sequence": 1224, "attempt_id": "4987e724-8150-449a-ac08-336caa82c0a0", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.177354063023813, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:03.642864+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:15:03.643Z" }, { "sequence": 1225, "attempt_id": "843bf84f-1cd1-4f37-8321-db0fdbeb39d1", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.8442152760690078, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:03.710502+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:15:03.710Z" }, { "sequence": 1226, "attempt_id": "45785f9e-9f9e-4aa9-a2dc-4175c6b887c5", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.6587127139791846, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:03.774439+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:15:03.774Z" }, { "sequence": 1227, "attempt_id": "24a52037-5d1e-4b4e-8b8b-08184518893a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1777842449955642, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:03.840665+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:15:03.842Z" }, { "sequence": 1228, "attempt_id": "27616aae-fa1a-4eda-b015-4d62e6755c68", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00535, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.633882159017958, "model_name": "gemini-3.1-pro-preview", "output_tokens": 260, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:03.923852+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00535, "created_at": "2026-09-17T03:15:03.923Z" }, { "sequence": 1229, "attempt_id": "9f46d937-1f16-4a82-9ff3-b96e47b62337", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2011405390221626, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:04.146145+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:15:04.146Z" }, { "sequence": 1230, "attempt_id": "60ad0a9c-ac25-4e29-bea8-39ca8a6060a5", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 7.176943654892966, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:04.213893+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T03:15:04.214Z" }, { "sequence": 1231, "attempt_id": "947d894c-4c44-48cc-8751-130942d16307", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.8413208499550819, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:04.484503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:15:04.484Z" }, { "sequence": 1232, "attempt_id": "7f4ebbfc-e912-4643-bbe2-d9fb2a35fcba", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00149025, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.467562102014199, "model_name": "gemini-3.8-flash", "output_tokens": 174, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:04.742368+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00149025, "created_at": "2026-09-17T03:15:04.742Z" }, { "sequence": 1233, "attempt_id": "369f985e-5e10-4bbd-b739-c2dd9a83ed6b", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.237291747936979, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:05.451520+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:15:05.451Z" }, { "sequence": 1234, "attempt_id": "7bbaf6ea-2fc9-470c-bf09-67a333cb9233", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040015, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.7658077260712162, "model_name": "muse-spark-1.2", "output_tokens": 203, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:05.604479+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0040015, "created_at": "2026-09-17T03:15:05.604Z" }, { "sequence": 1235, "attempt_id": "6cf2075e-1d9c-44bf-99fa-f56466c6db9b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2054606350138783, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:05.757075+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:15:05.757Z" }, { "sequence": 1236, "attempt_id": "7037b751-a0e8-4577-b4f6-4603c33a15e1", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006252, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.9759991340106353, "model_name": "claude-sonnet-5", "output_tokens": 60, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:05.964531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006252, "created_at": "2026-09-17T03:15:05.964Z" }, { "sequence": 1237, "attempt_id": "fb635e5d-74d3-4a5f-96a3-bb1bdbded703", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.123782537993975, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:06.032589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:15:06.033Z" }, { "sequence": 1238, "attempt_id": "12c15b7c-a66f-49e7-ad53-a0712ae47a72", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2328114500269294, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:06.097620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:15:06.099Z" }, { "sequence": 1239, "attempt_id": "f721d712-756d-4785-8190-d85d1b04a87c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00421825, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.3901778259314597, "model_name": "muse-spark-1.3", "output_tokens": 254, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:06.164438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00421825, "created_at": "2026-09-17T03:15:06.164Z" }, { "sequence": 1240, "attempt_id": "70b3bc7d-5b9b-4abe-a5fd-025e7c07440a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.962256320985034, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:06.240990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:15:06.241Z" }, { "sequence": 1241, "attempt_id": "ea4474c6-9475-493b-9c92-cd5f7ef2a6e2", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.3862357520265505, "model_name": "gemini-3.1-pro-preview", "output_tokens": 116, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:06.554750+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003626, "created_at": "2026-09-17T03:15:06.554Z" }, { "sequence": 1242, "attempt_id": "40c4c324-a436-44fe-b589-e1d82f9fc009", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.7538511659950018, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:06.786733+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003561, "created_at": "2026-09-17T03:15:06.786Z" }, { "sequence": 1243, "attempt_id": "73d4a670-d43c-4461-ae1b-3b13f9901c77", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.708415818051435, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:07.228295+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:15:07.228Z" }, { "sequence": 1244, "attempt_id": "2bfc60ee-970f-4c1a-8c5f-05b4c730174e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.5396450450643897, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:07.380564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:15:07.380Z" }, { "sequence": 1245, "attempt_id": "c08403ac-f1ba-4f47-9773-f20464082207", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00141525, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.7617136519402266, "model_name": "gemini-3.8-flash", "output_tokens": 150, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:07.584466+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00141525, "created_at": "2026-09-17T03:15:07.584Z" }, { "sequence": 1246, "attempt_id": "76354ce4-0985-449a-a9a0-edcd969c04e1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.8098154990002513, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:07.664390+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:15:07.664Z" }, { "sequence": 1247, "attempt_id": "f41473ab-fc46-4368-a2ea-f3df79349667", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.5628063899930567, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:08.185335+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:15:08.185Z" }, { "sequence": 1248, "attempt_id": "1315e1dd-881a-4915-b841-add609aee427", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00472775, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.264912256039679, "model_name": "muse-spark-1.2", "output_tokens": 368, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:08.429634+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00472775, "created_at": "2026-09-17T03:15:08.429Z" }, { "sequence": 1249, "attempt_id": "097d20b6-5f36-42c1-9918-8ab307876abb", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.2099071469856426, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:08.514898+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:15:08.515Z" }, { "sequence": 1250, "attempt_id": "9bc63174-5d57-4596-a459-be09204f2c44", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003978, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.844536190968938, "model_name": "gemini-3.1-pro-preview", "output_tokens": 142, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:08.579319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003978, "created_at": "2026-09-17T03:15:08.579Z" }, { "sequence": 1251, "attempt_id": "710a4658-a5cb-49e2-a487-da923b617a69", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.106015807017684, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:08.644724+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:15:08.644Z" }, { "sequence": 1252, "attempt_id": "5bd8bf55-ada7-4f8f-b595-11939d596d31", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9661910269642249, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:08.712059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:15:08.713Z" }, { "sequence": 1253, "attempt_id": "bd027342-84b2-48b7-bd63-33bb8373e9ed", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0483158500865102, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:08.777378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:15:08.778Z" }, { "sequence": 1254, "attempt_id": "6484e6b7-5cbc-4079-a60f-c70edf322bf1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004443, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.8763882409548387, "model_name": "muse-spark-1.3", "output_tokens": 301, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:08.974274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004443, "created_at": "2026-09-17T03:15:08.974Z" }, { "sequence": 1255, "attempt_id": "1b8597b3-50f6-4e84-993a-1de74c5b0f60", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.2853322960436344, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:09.580526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:15:09.580Z" }, { "sequence": 1256, "attempt_id": "fd3c5cec-161b-47d9-951c-1dd5bc81970e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.3854733059415594, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:09.965128+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:15:09.965Z" }, { "sequence": 1257, "attempt_id": "c02c3efa-a3ba-46f3-b1df-945b1fd00646", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00384, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.5651538070524111, "model_name": "muse-spark-1.2", "output_tokens": 165, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:10.277551+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00384, "created_at": "2026-09-17T03:15:10.277Z" }, { "sequence": 1258, "attempt_id": "b4633d83-b6e4-4275-babe-252758ad07c4", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00124275, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.8006067739333957, "model_name": "gemini-3.8-flash", "output_tokens": 108, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:10.347010+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00124275, "created_at": "2026-09-17T03:15:10.347Z" }, { "sequence": 1259, "attempt_id": "1bdf6519-856a-4b74-a27a-97655f418bb9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040185, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.816687901969999, "model_name": "muse-spark-1.3", "output_tokens": 207, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:10.526927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0040185, "created_at": "2026-09-17T03:15:10.527Z" }, { "sequence": 1260, "attempt_id": "17d0d83e-fe79-4c69-8d62-513afe132598", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.2484773509204388, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:10.896883+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:15:10.897Z" }, { "sequence": 1261, "attempt_id": "09f146a3-3db0-4172-9bd9-276eda9655c5", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006458, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.1829612720757723, "model_name": "claude-sonnet-5", "output_tokens": 78, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:11.223863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006458, "created_at": "2026-09-17T03:15:11.224Z" }, { "sequence": 1262, "attempt_id": "d63befa1-c35b-419f-b977-49309cdd024b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.7747727159876376, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:11.368619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004968, "created_at": "2026-09-17T03:15:11.368Z" }, { "sequence": 1263, "attempt_id": "0f5f7880-955d-4f95-adb9-caa417a58236", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6129046909045428, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:11.646637+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:15:11.646Z" }, { "sequence": 1264, "attempt_id": "754634e7-8abc-4d14-a836-afdd7e464967", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01652, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.937249858980067, "model_name": "claude-opus-5", "output_tokens": 93, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:11.780003+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01652, "created_at": "2026-09-17T03:15:11.780Z" }, { "sequence": 1265, "attempt_id": "710add76-30c2-4f60-adf5-1176eaf5c834", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00455, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.5501837680349126, "model_name": "gemini-3.1-pro-preview", "output_tokens": 193, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:11.848635+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00455, "created_at": "2026-09-17T03:15:11.850Z" }, { "sequence": 1266, "attempt_id": "41da1914-bd10-49d0-9025-51a2b69037bc", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.365368698956445, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:12.143051+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:15:12.143Z" }, { "sequence": 1267, "attempt_id": "d33146c6-3c1f-40e0-8391-f64df95db394", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7198542709229514, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:12.212488+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:15:12.212Z" }, { "sequence": 1268, "attempt_id": "332dd20d-b1f5-4934-a37c-3fc4f31b1d0a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005448, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.86307211604435, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:12.278938+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005448, "created_at": "2026-09-17T03:15:12.279Z" }, { "sequence": 1269, "attempt_id": "97a746c3-2a27-41e5-bf04-a8600d11a22a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.137911804020405, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:12.485236+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:15:12.485Z" }, { "sequence": 1270, "attempt_id": "369f9c79-d84c-433f-88a8-1155f0077228", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.8553624099586159, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:13.200565+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001856, "created_at": "2026-09-17T03:15:13.200Z" }, { "sequence": 1271, "attempt_id": "bb0cf37c-c65a-42f8-8c09-3fad64edaf2f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00198375, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.34392602997832, "model_name": "gemini-3.8-flash", "output_tokens": 299, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:13.636223+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00198375, "created_at": "2026-09-17T03:15:13.636Z" }, { "sequence": 1272, "attempt_id": "79f45a8b-65d5-4172-80c6-1744db93584b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005344, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.5986394779756665, "model_name": "gpt-5.6-terra", "output_tokens": 48, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:15.304514+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005344, "created_at": "2026-09-17T03:15:15.304Z" }, { "sequence": 1273, "attempt_id": "d9161048-90e6-409c-895e-82bb7c707131", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017255, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.2914304269943386, "model_name": "claude-opus-5", "output_tokens": 125, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:15.504311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017255, "created_at": "2026-09-17T03:15:15.504Z" }, { "sequence": 1274, "attempt_id": "87136214-ac81-4323-890a-6e54bb2351a9", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02654, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.0680234589381143, "model_name": "gpt-6-astra", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:15.626402+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02654, "created_at": "2026-09-17T03:15:15.628Z" }, { "sequence": 1275, "attempt_id": "43bc0a75-8f72-4103-af57-9384e8d96aaf", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00584, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.657434228924103, "model_name": "gemini-3.1-pro-preview", "output_tokens": 295, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:15.710243+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00584, "created_at": "2026-09-17T03:15:15.710Z" }, { "sequence": 1276, "attempt_id": "9a99d589-fa6f-4547-9c52-dca5e8204894", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003543, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.7549052060348913, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:16.556966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003543, "created_at": "2026-09-17T03:15:16.557Z" }, { "sequence": 1277, "attempt_id": "dd5df510-f6ea-417b-99e4-68c1d8e278ad", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02903, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.157988767954521, "model_name": "claude-fable-5-1", "output_tokens": 15, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:17.082605+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02903, "created_at": "2026-09-17T03:15:17.082Z" }, { "sequence": 1278, "attempt_id": "f15cc972-deaf-4c24-8adc-ffa4c2efeee1", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 11.1389847319806, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:17.369161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:15:17.369Z" }, { "sequence": 1279, "attempt_id": "e597251b-5f41-48ec-a0b7-719a661b6317", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006432, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 5.360130065935664, "model_name": "claude-sonnet-5", "output_tokens": 78, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:17.639445+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006432, "created_at": "2026-09-17T03:15:17.639Z" }, { "sequence": 1280, "attempt_id": "e5241486-cdf0-486b-8064-0feb681c3685", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00808925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.940864754025824, "model_name": "muse-spark-1.2", "output_tokens": 1156, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:17.789846+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00808925, "created_at": "2026-09-17T03:15:17.790Z" }, { "sequence": 1281, "attempt_id": "bdacfa9e-073b-461e-976c-3db11d8a00ef", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005644, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.7708455909742042, "model_name": "gpt-5.6-luna", "output_tokens": 73, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:18.165915+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005644, "created_at": "2026-09-17T03:15:18.166Z" }, { "sequence": 1282, "attempt_id": "345c59ab-6700-43fd-884b-aa90acc62017", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00668675, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 7.742811545962468, "model_name": "muse-spark-1.3", "output_tokens": 826, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:19.458162+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00668675, "created_at": "2026-09-17T03:15:19.458Z" }, { "sequence": 1283, "attempt_id": "65e57cc8-d4a4-42c7-93cb-5d25cb5820b6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.4027256059926003, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:15:19.689571+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:15:19.689Z" }, { "sequence": 1284, "attempt_id": "9da1ce45-345b-4867-a1af-167d61fd056b", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00505575, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 4.255918047041632, "model_name": "gemini-3.8-flash", "output_tokens": 1122, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:20.054780+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00505575, "created_at": "2026-09-17T03:15:20.054Z" }, { "sequence": 1285, "attempt_id": "9c1ee6aa-9836-4cd3-b062-95c94835aeab", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.7571636049542576, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:15:21.255457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:15:21.255Z" }, { "sequence": 1286, "attempt_id": "78758829-15f3-4df3-bac2-a86b3d250460", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.8806518040364608, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:21.410843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:15:21.411Z" }, { "sequence": 1287, "attempt_id": "f19db9f1-eb1e-4ea4-9c74-2958d260e3c6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.013636, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 8.210190237034112, "model_name": "gpt-5.6-sol", "output_tokens": 205, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:21.483066+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.013636, "created_at": "2026-09-17T03:15:21.483Z" }, { "sequence": 1288, "attempt_id": "cd0311ae-28b3-47f8-8f54-e50202eeab9c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008778, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 4.1550521169556305, "model_name": "claude-sonnet-5", "output_tokens": 291, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:22.016524+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008778, "created_at": "2026-09-17T03:15:22.016Z" }, { "sequence": 1289, "attempt_id": "d9c0d0eb-5817-4f7d-b064-8ba88c7efe86", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8386752229416743, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:15:22.930098+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:15:22.930Z" }, { "sequence": 1290, "attempt_id": "64428181-aa79-4877-be28-8dd8d4e64249", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011624, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.346929550054483, "model_name": "gpt-5.6-sol", "output_tokens": 88, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:23.108948+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011624, "created_at": "2026-09-17T03:15:23.109Z" }, { "sequence": 1291, "attempt_id": "1f0d604d-edb1-496c-bd92-90f345e51a42", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006588, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.0746574950171635, "model_name": "gpt-5.6-terra", "output_tokens": 138, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:23.200670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006588, "created_at": "2026-09-17T03:15:23.200Z" }, { "sequence": 1292, "attempt_id": "f30f21fc-6dab-4d66-b84c-b8b33a052678", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00152475, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.7785426808986813, "model_name": "gemini-3.8-flash", "output_tokens": 164, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:15:23.331668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00152475, "created_at": "2026-09-17T03:15:23.331Z" }, { "sequence": 1293, "attempt_id": "d564c417-717f-458b-bc7a-c7d7f42ece48", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.022745, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.977127359015867, "model_name": "claude-opus-5", "output_tokens": 323, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:23.693348+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.022745, "created_at": "2026-09-17T03:15:23.693Z" }, { "sequence": 1294, "attempt_id": "c47902c0-e997-4fd4-85ed-da5c008a2426", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006192000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.3755290070548654, "model_name": "gpt-5.6-luna", "output_tokens": 105, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:23.764988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006192000000000001, "created_at": "2026-09-17T03:15:23.766Z" }, { "sequence": 1295, "attempt_id": "ff4b594d-f548-40a4-b826-326947dfd8cc", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.7073399859946221, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:24.544778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T03:15:24.544Z" }, { "sequence": 1296, "attempt_id": "d97299ab-b2a6-443d-aac8-b3fcf86773d1", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.1315712248906493, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:25.540589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01434, "created_at": "2026-09-17T03:15:25.540Z" }, { "sequence": 1297, "attempt_id": "33adb08a-9af6-4a43-879e-2321f0f081ba", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.9309732450637966, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:25.696276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005736, "created_at": "2026-09-17T03:15:25.696Z" }, { "sequence": 1298, "attempt_id": "3868e6a5-e376-4006-b3d4-a0648ed2abae", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.015378, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 10.138744972995482, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1093, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:25.768012+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015378, "created_at": "2026-09-17T03:15:25.769Z" }, { "sequence": 1299, "attempt_id": "7b40b3cd-7bbb-4d97-afc6-a63c962e735b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.1921137870522216, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:26.808166+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:15:26.808Z" }, { "sequence": 1300, "attempt_id": "573be90f-060e-4ff3-bf25-63af752d8ee3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.2920110240811482, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:26.904951+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:15:26.905Z" }, { "sequence": 1301, "attempt_id": "a9319b99-3ac5-4767-807d-56139401c1a5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005326, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.354639911907725, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:27.122962+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005326, "created_at": "2026-09-17T03:15:27.123Z" }, { "sequence": 1302, "attempt_id": "4b1967f1-495f-4224-9f73-d22b95271790", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.861141834058799, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:27.194003+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T03:15:27.194Z" }, { "sequence": 1303, "attempt_id": "fe80199b-b583-4c76-a996-0ca186e2d218", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.5320883890381083, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:27.381614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:15:27.381Z" }, { "sequence": 1304, "attempt_id": "6226427b-c04a-452e-b8c6-566f2f8b8786", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00753, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 4.5226199850440025, "model_name": "muse-spark-1.2", "output_tokens": 1005, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:15:27.705044+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00753, "created_at": "2026-09-17T03:15:27.705Z" }, { "sequence": 1305, "attempt_id": "54f6b423-9353-4c3a-a47e-7f03ea0fd4f1", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.818640039069578, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:15:28.012953+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:15:28.013Z" }, { "sequence": 1306, "attempt_id": "4f856c2f-d6b3-42f3-9ce0-be222743dede", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001335, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.2842700988985598, "model_name": "gemini-3.8-flash", "output_tokens": 128, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:15:28.260126+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001335, "created_at": "2026-09-17T03:15:28.260Z" }, { "sequence": 1307, "attempt_id": "eda522ec-8232-4166-a1d6-56b589ae22b6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.008087, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 11.726426254026592, "model_name": "muse-spark-1.3", "output_tokens": 1159, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:28.357908+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008087, "created_at": "2026-09-17T03:15:28.358Z" }, { "sequence": 1308, "attempt_id": "b900874c-8a5a-4c46-8423-fd937ebb9a97", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.7205281379865482, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:29.147676+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:15:29.147Z" }, { "sequence": 1309, "attempt_id": "0fc7caf9-dbc9-4b7b-96f1-2ed091d97421", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044905, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 2.0136960180243477, "model_name": "muse-spark-1.2", "output_tokens": 311, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:29.466876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0044905, "created_at": "2026-09-17T03:15:29.467Z" }, { "sequence": 1310, "attempt_id": "8b883e32-8571-4302-8ddc-8114f4c3fa42", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012686, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.381633977987804, "model_name": "gemini-3.1-pro-preview", "output_tokens": 855, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:15:29.865006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012686, "created_at": "2026-09-17T03:15:29.865Z" }, { "sequence": 1311, "attempt_id": "d2e012d1-8753-48c8-8928-3d225713d54d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004464, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.321522808051668, "model_name": "gemini-3.1-pro-preview", "output_tokens": 182, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:30.201483+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004464, "created_at": "2026-09-17T03:15:30.201Z" }, { "sequence": 1312, "attempt_id": "5de45a43-4a35-4eb7-b580-634a2fd0fcbc", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004601, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.0402124479878694, "model_name": "muse-spark-1.3", "output_tokens": 337, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:30.303094+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004601, "created_at": "2026-09-17T03:15:30.303Z" }, { "sequence": 1313, "attempt_id": "6245d0f6-0df6-4489-95c7-7b21c302f50d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01642, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.6502997409552336, "model_name": "claude-opus-5", "output_tokens": 89, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:30.733723+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01642, "created_at": "2026-09-17T03:15:30.733Z" }, { "sequence": 1314, "attempt_id": "6b3a5ae8-08a3-4097-bbfc-bedb4748864b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6502706829924136, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:30.869548+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:15:30.869Z" }, { "sequence": 1315, "attempt_id": "46e80179-7e3b-4535-96cd-0f1fa84602d2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00726225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 7.896515092928894, "model_name": "muse-spark-1.3", "output_tokens": 942, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:30.952568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00726225, "created_at": "2026-09-17T03:15:30.952Z" }, { "sequence": 1316, "attempt_id": "9f0b39ee-b5ac-4e52-9709-6e1b8f95a7f1", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006858, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.8583443220704794, "model_name": "claude-sonnet-5", "output_tokens": 118, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:31.189006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006858, "created_at": "2026-09-17T03:15:31.189Z" }, { "sequence": 1317, "attempt_id": "a994ae29-638b-4385-85ec-5097442b5ec0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.030184431001544, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:31.299628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004968, "created_at": "2026-09-17T03:15:31.299Z" }, { "sequence": 1318, "attempt_id": "61e99d63-e14d-46ad-b04f-4aab618946c2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.113057394977659, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:31.648720+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:15:31.648Z" }, { "sequence": 1319, "attempt_id": "b97affdb-46a6-47da-a336-df9c712719ee", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.941538208979182, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:31.722134+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:15:31.722Z" }, { "sequence": 1320, "attempt_id": "36967e90-50e0-4ee7-bf42-6b10acb624ab", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9217612019274384, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:31.874703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:15:31.874Z" }, { "sequence": 1321, "attempt_id": "a88c12d0-d276-4a67-bf8b-82f49b1da66f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005592, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.1344363620737568, "model_name": "gpt-5.6-terra", "output_tokens": 66, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:32.068947+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005592, "created_at": "2026-09-17T03:15:32.069Z" }, { "sequence": 1322, "attempt_id": "fc017509-4737-42fd-ae43-c9923f32c210", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01450875, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 15.108454669010825, "model_name": "muse-spark-1.2", "output_tokens": 2670, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:32.264489+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01450875, "created_at": "2026-09-17T03:15:32.264Z" }, { "sequence": 1323, "attempt_id": "57116497-5b42-4263-a2db-b638316211c4", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 1.0331482499605045, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:32.980959+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T03:15:32.981Z" }, { "sequence": 1324, "attempt_id": "eaf5ed87-490d-4d45-8869-85d60af36369", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003548, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.0901297860546038, "model_name": "gemini-3.1-pro-preview", "output_tokens": 104, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:33.460608+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003548, "created_at": "2026-09-17T03:15:33.460Z" }, { "sequence": 1325, "attempt_id": "3fedf498-ec50-4276-9b9e-b47953a3a177", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014345, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.3617012510076165, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:34.084168+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014345, "created_at": "2026-09-17T03:15:34.084Z" }, { "sequence": 1326, "attempt_id": "85c27e8d-d2f1-4621-8aa3-62c4578ca289", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00184125, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.5397661520401016, "model_name": "gemini-3.8-flash", "output_tokens": 261, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:34.344756+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00184125, "created_at": "2026-09-17T03:15:34.344Z" }, { "sequence": 1327, "attempt_id": "69fb2541-9018-4822-a810-2b92fd21fbce", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005234, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.4730375970248133, "model_name": "gpt-5.6-terra", "output_tokens": 38, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:34.526480+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005234, "created_at": "2026-09-17T03:15:34.526Z" }, { "sequence": 1328, "attempt_id": "8b2c5444-4b1a-4c9c-a4fe-a08879304d44", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.1532358509721234, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:34.685594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:15:34.685Z" }, { "sequence": 1329, "attempt_id": "0830a0a8-47d1-4889-8d54-e71d053f8ea8", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.328210109961219, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:34.754364+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T03:15:34.754Z" }, { "sequence": 1330, "attempt_id": "e42c714b-f9cc-438b-bfff-43e5e7b322e9", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006698, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 3.395356275024824, "model_name": "claude-sonnet-5", "output_tokens": 107, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:35.189977+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006698, "created_at": "2026-09-17T03:15:35.190Z" }, { "sequence": 1331, "attempt_id": "5682f804-0a2c-46b6-b157-e2bef6adfc1d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 3.250095516908914, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:35.392474+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:15:35.392Z" }, { "sequence": 1332, "attempt_id": "2d943991-2dd7-43ce-b64e-eb17969c0557", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00663575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.198105251998641, "model_name": "muse-spark-1.2", "output_tokens": 814, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:35.464561+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00663575, "created_at": "2026-09-17T03:15:35.466Z" }, { "sequence": 1333, "attempt_id": "041b85b5-7fde-45ac-bcdd-5a21940e8e12", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.4749060830799863, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:36.069612+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003683, "created_at": "2026-09-17T03:15:36.069Z" }, { "sequence": 1334, "attempt_id": "32fc2498-436f-4ca1-a041-dcb8c786bab7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.296266062068753, "model_name": "muse-spark-1.3", "output_tokens": 545, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:36.321052+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0054925, "created_at": "2026-09-17T03:15:36.321Z" }, { "sequence": 1335, "attempt_id": "0b3c1f38-5505-4641-abed-e4ab7087fd59", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00238575, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.405517528997734, "model_name": "gemini-3.8-flash", "output_tokens": 409, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:36.818931+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00238575, "created_at": "2026-09-17T03:15:36.819Z" }, { "sequence": 1336, "attempt_id": "4d5eeedd-d337-4b19-bc2a-e331b06edc21", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.6989776190603152, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:36.915776+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001856, "created_at": "2026-09-17T03:15:36.916Z" }, { "sequence": 1337, "attempt_id": "1c0ae9f6-36ec-465e-a9e6-01edb805df91", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.6678674010327086, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:38.060635+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02454, "created_at": "2026-09-17T03:15:38.060Z" }, { "sequence": 1338, "attempt_id": "203dcc9c-729e-4b52-8900-f5b62c769cd0", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006882, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.5568224999587983, "model_name": "claude-sonnet-5", "output_tokens": 123, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:38.136046+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006882, "created_at": "2026-09-17T03:15:38.137Z" }, { "sequence": 1339, "attempt_id": "9697c080-d81c-422b-b2e6-b4f0070706bc", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01598, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.0001188169699162, "model_name": "claude-opus-5", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:38.465008+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01598, "created_at": "2026-09-17T03:15:38.465Z" }, { "sequence": 1340, "attempt_id": "2744b2bb-ca8b-4f3c-86d6-80aae84a3331", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 6.291032914072275, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:38.628888+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:15:38.629Z" }, { "sequence": 1341, "attempt_id": "aa74bf7b-5013-4195-92de-21c7f28e4081", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010796, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.724334905971773, "model_name": "gpt-5.6-sol", "output_tokens": 63, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:38.703650+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010796, "created_at": "2026-09-17T03:15:38.703Z" }, { "sequence": 1342, "attempt_id": "d1bf31dc-566f-420d-9380-0e38354b5227", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005452, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.6408740659244359, "model_name": "gpt-5.6-terra", "output_tokens": 57, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:38.774911+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005452, "created_at": "2026-09-17T03:15:38.775Z" }, { "sequence": 1343, "attempt_id": "393a86b3-1300-417a-8bdf-56419025b116", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0045395, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.9804895650595427, "model_name": "muse-spark-1.2", "output_tokens": 324, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:38.850456+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0045395, "created_at": "2026-09-17T03:15:38.851Z" }, { "sequence": 1344, "attempt_id": "08b3e332-4ee0-4424-9595-92b1475ab627", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006412, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 5.034085288993083, "model_name": "gemini-3.1-pro-preview", "output_tokens": 345, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:39.190043+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006412, "created_at": "2026-09-17T03:15:39.190Z" }, { "sequence": 1345, "attempt_id": "f4d2c833-3fdc-4db0-b8e5-71db0ef18012", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.8625750930514187, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:39.566562+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003668, "created_at": "2026-09-17T03:15:39.566Z" }, { "sequence": 1346, "attempt_id": "c85867a1-4933-4a91-9519-a27d960da0c0", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00582725, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 5.971923682023771, "model_name": "muse-spark-1.3", "output_tokens": 627, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:40.726646+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00582725, "created_at": "2026-09-17T03:15:40.726Z" }, { "sequence": 1347, "attempt_id": "12a67fbc-5ca4-4097-8d90-1c61482bb58e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02963, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.700544994091615, "model_name": "claude-fable-5-1", "output_tokens": 27, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:40.972501+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02963, "created_at": "2026-09-17T03:15:40.972Z" }, { "sequence": 1348, "attempt_id": "b061679a-07d0-470b-8c96-8c4fb06ff54a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005776, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.163878130959347, "model_name": "gpt-5.6-luna", "output_tokens": 84, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:41.300357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005776, "created_at": "2026-09-17T03:15:41.300Z" }, { "sequence": 1349, "attempt_id": "1c36cc3e-7edc-4296-b1d0-f5bb7eb36f84", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6296606309479102, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:41.441504+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T03:15:41.441Z" }, { "sequence": 1350, "attempt_id": "6c877044-bc3a-4e29-8f4c-85dcbf76a74b", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.1226355229737237, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:41.763667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T03:15:41.763Z" }, { "sequence": 1351, "attempt_id": "1a389071-04a5-4268-bc94-ded4af1e7b20", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038295, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 3.383284442941658, "model_name": "gemini-3.8-flash", "output_tokens": 795, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:41.921669+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0038295, "created_at": "2026-09-17T03:15:41.921Z" }, { "sequence": 1352, "attempt_id": "39426453-e846-4088-af7f-8c09319fbea4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2685214600060135, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:42.647421+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:15:42.647Z" }, { "sequence": 1353, "attempt_id": "18c6baf2-b8b9-41f4-b7d9-a5d9837b8a17", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6768909769598395, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:42.733620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:15:42.733Z" }, { "sequence": 1354, "attempt_id": "471019aa-6af1-48fb-bed0-7aa955ad479b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9086948609910905, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:42.810820+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:15:42.811Z" }, { "sequence": 1355, "attempt_id": "50555940-eeaa-432f-b605-c7e25585c980", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018065, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 3.5916036850539967, "model_name": "claude-opus-5", "output_tokens": 161, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:42.889470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.018065, "created_at": "2026-09-17T03:15:42.891Z" }, { "sequence": 1356, "attempt_id": "0cce1588-0d63-4643-b2d6-003aa934b452", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.8932333999546245, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:42.966797+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T03:15:42.966Z" }, { "sequence": 1357, "attempt_id": "03f01b6c-992e-4456-8dec-908d5e28f6d2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00544, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.4448895490495488, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:43.042922+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00544, "created_at": "2026-09-17T03:15:43.043Z" }, { "sequence": 1358, "attempt_id": "3d2cd3a8-f362-42de-9e0c-6920bb90fff5", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.7138833759818226, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:43.525082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:15:43.525Z" }, { "sequence": 1359, "attempt_id": "03b35386-d9f5-4e5b-86b3-8d598ca19c3b", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00993, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 6.776845525950193, "model_name": "gemini-3.1-pro-preview", "output_tokens": 639, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:44.987667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00993, "created_at": "2026-09-17T03:15:44.987Z" }, { "sequence": 1360, "attempt_id": "a8634089-e337-4a76-95ba-ed821d53b2eb", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.019916226970963, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:45.144149+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:15:45.144Z" }, { "sequence": 1361, "attempt_id": "66e5ecaa-45cb-401d-9161-e9f168ad367f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00268275, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.419223807984963, "model_name": "gemini-3.8-flash", "output_tokens": 488, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:45.218155+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00268275, "created_at": "2026-09-17T03:15:45.219Z" }, { "sequence": 1362, "attempt_id": "13cf571c-96ca-403f-8d63-ebcf51f5b9b9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00881375, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 6.720163014018908, "model_name": "muse-spark-1.2", "output_tokens": 1330, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:45.570990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00881375, "created_at": "2026-09-17T03:15:45.571Z" }, { "sequence": 1363, "attempt_id": "dcf50596-a699-4bd2-8623-eb188176f6a4", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.6312154530314729, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:45.692306+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:15:45.692Z" }, { "sequence": 1364, "attempt_id": "22505c9a-6bad-45df-a520-eff3ccfb5743", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.177576476940885, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:45.779966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:15:45.780Z" }, { "sequence": 1365, "attempt_id": "c8d01e0c-1dbd-40d5-adec-30ced8c4ea56", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.366358311031945, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:46.409594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:15:46.409Z" }, { "sequence": 1366, "attempt_id": "f11cf7a6-219a-445d-bc21-40d91f75a65c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.3121247930685058, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:46.604542+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:15:46.604Z" }, { "sequence": 1367, "attempt_id": "b8998cfc-24df-458a-9519-6ff29cda468a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9728321810252964, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:46.677620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:15:46.677Z" }, { "sequence": 1368, "attempt_id": "33b82176-c09a-45d0-9561-2c4596fe2b19", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006183, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 7.8986429730430245, "model_name": "muse-spark-1.3", "output_tokens": 711, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:46.749018+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006183, "created_at": "2026-09-17T03:15:46.749Z" }, { "sequence": 1369, "attempt_id": "a59352c4-9d40-4761-ac7b-9e9cebe6886c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0086786360479891, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:46.821549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:15:46.821Z" }, { "sequence": 1370, "attempt_id": "9b4685dd-3472-444a-bdab-bcc0a54419dc", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.761692548985593, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:47.439626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:15:47.439Z" }, { "sequence": 1371, "attempt_id": "ad27598d-7e7f-4ee4-bea4-ae872d225bdf", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00123525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.0431890080217272, "model_name": "gemini-3.8-flash", "output_tokens": 106, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:47.526155+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00123525, "created_at": "2026-09-17T03:15:47.526Z" }, { "sequence": 1372, "attempt_id": "7523004e-890a-4977-a1f4-a530ed31b5c6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.4856990430271253, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:47.704197+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:15:47.704Z" }, { "sequence": 1373, "attempt_id": "c45c28e0-69bf-47c3-b11d-aba20f2dcf7c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038995, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.2975182599620894, "model_name": "muse-spark-1.2", "output_tokens": 179, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:48.119423+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0038995, "created_at": "2026-09-17T03:15:48.119Z" }, { "sequence": 1374, "attempt_id": "4a856ecd-69fc-4658-90fd-b84089182467", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.5576154210139066, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:48.335943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T03:15:48.336Z" }, { "sequence": 1375, "attempt_id": "5e03b5ee-6e9d-4d72-8a4a-64da2796ddc0", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010326, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 6.693077166099101, "model_name": "gemini-3.1-pro-preview", "output_tokens": 671, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:48.744860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010326, "created_at": "2026-09-17T03:15:48.745Z" }, { "sequence": 1376, "attempt_id": "b3f4fa21-02eb-4a55-9c05-97de0c5eed76", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004214, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.399967946927063, "model_name": "gemini-3.1-pro-preview", "output_tokens": 165, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:49.253958+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004214, "created_at": "2026-09-17T03:15:49.254Z" }, { "sequence": 1377, "attempt_id": "32577283-f446-487e-8c78-5c26031cf4d4", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.7495486439438537, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:49.348588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:15:49.348Z" }, { "sequence": 1378, "attempt_id": "99ba2738-8cd8-4026-bb6b-701ab2c816b2", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.0180706990649924, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:49.531284+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T03:15:49.531Z" }, { "sequence": 1379, "attempt_id": "1b393057-2d05-49ed-8c4e-fed72b7eb6d4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.2448059059679508, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:49.657343+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:15:49.657Z" }, { "sequence": 1380, "attempt_id": "14cb8ad5-2a0b-4f66-95fb-71a51f3f0728", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0153830719646066, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:49.868598+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:15:49.868Z" }, { "sequence": 1381, "attempt_id": "84b92b37-51b3-4f9a-bca0-16a21b890f8e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.8433062949916348, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:50.037840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:15:50.038Z" }, { "sequence": 1382, "attempt_id": "60ca5067-e328-4526-89cd-6808d4d83e02", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0045625, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.3197812070138752, "model_name": "muse-spark-1.3", "output_tokens": 335, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:50.116979+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0045625, "created_at": "2026-09-17T03:15:50.117Z" }, { "sequence": 1383, "attempt_id": "9553f025-b7af-452b-a657-74bb47a4f935", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0050814049318433, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:50.338897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:15:50.339Z" }, { "sequence": 1384, "attempt_id": "f531ea31-9783-4387-963e-222ce1825823", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.9461855919798836, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:50.735389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:15:50.735Z" }, { "sequence": 1385, "attempt_id": "2fb70a55-ba27-47ee-bab0-db643ab454d0", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 4.137856925022788, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:51.031051+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T03:15:51.031Z" }, { "sequence": 1386, "attempt_id": "8a7c686e-d1b2-480b-bcd8-7c821039d5af", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00375975, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.0948480110382661, "model_name": "muse-spark-1.2", "output_tokens": 147, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:51.212148+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00375975, "created_at": "2026-09-17T03:15:51.212Z" }, { "sequence": 1387, "attempt_id": "6d15f7cd-c83d-4072-b485-a515b524b9e2", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001293, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 2.141020640032366, "model_name": "gemini-3.8-flash", "output_tokens": 122, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:51.756820+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001293, "created_at": "2026-09-17T03:15:51.756Z" }, { "sequence": 1388, "attempt_id": "b1c7e28c-e1b6-43c7-8014-5386fa7dc0a2", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-024", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.6640378150623292, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:51.833687+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:15:51.833Z" }, { "sequence": 1389, "attempt_id": "b74d2160-9a9b-46b0-ba29-d63be9e5808d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00394675, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 2.247738915029913, "model_name": "muse-spark-1.3", "output_tokens": 191, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:52.214828+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00394675, "created_at": "2026-09-17T03:15:52.215Z" }, { "sequence": 1390, "attempt_id": "0721568b-255e-4aa0-a7cf-9419d18befa4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005384, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.5971743060508743, "model_name": "gemini-3.1-pro-preview", "output_tokens": 263, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:53.024896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005384, "created_at": "2026-09-17T03:15:53.025Z" }, { "sequence": 1391, "attempt_id": "0f4d8129-a3cd-4483-aa6f-ef672a983f16", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00582, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7953735580667853, "model_name": "gpt-5.6-terra", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:53.708198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00582, "created_at": "2026-09-17T03:15:53.708Z" }, { "sequence": 1392, "attempt_id": "d068146b-e65a-426f-85f3-de8c8cdd45d9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006036, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.631417185999453, "model_name": "gpt-5.6-luna", "output_tokens": 92, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:53.923259+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006036, "created_at": "2026-09-17T03:15:53.923Z" }, { "sequence": 1393, "attempt_id": "956220b3-2f2b-4d94-874b-6c6f6817d54e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007338, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.6746733649633825, "model_name": "claude-sonnet-5", "output_tokens": 147, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:54.484832+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007338, "created_at": "2026-09-17T03:15:54.484Z" }, { "sequence": 1394, "attempt_id": "18d0d23b-4658-4988-8f84-f39d113675d9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-024", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7619712670566514, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:15:54.760132+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:15:54.760Z" }, { "sequence": 1395, "attempt_id": "8812aa9b-4522-434f-952e-87c3087dfaf2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011264, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.6082303690491244, "model_name": "gpt-5.6-sol", "output_tokens": 70, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:55.442273+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011264, "created_at": "2026-09-17T03:15:55.442Z" }, { "sequence": 1396, "attempt_id": "1ad713a8-47a6-4483-a1bf-58a25c33bd17", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-024", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.03021, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 5.382723885006271, "model_name": "claude-fable-5-1", "output_tokens": 17, "prediction": { "choice": "A" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:55.583082+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03021, "created_at": "2026-09-17T03:15:55.583Z" }, { "sequence": 1397, "attempt_id": "2d6478fd-9528-4ad0-a9a0-f066606b4fb9", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0026835, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.9833101710537449, "model_name": "gemini-3.8-flash", "output_tokens": 473, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:55.767384+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0026835, "created_at": "2026-09-17T03:15:55.767Z" }, { "sequence": 1398, "attempt_id": "7dc4906f-62c6-41db-886c-f9246638c5a4", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.020095, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.548398558981717, "model_name": "claude-opus-5", "output_tokens": 217, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:55.964677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.020095, "created_at": "2026-09-17T03:15:55.964Z" }, { "sequence": 1399, "attempt_id": "36226aa8-9b58-475d-a935-29f0a7d2a854", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.7802846509730443, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:56.821775+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T03:15:56.821Z" }, { "sequence": 1400, "attempt_id": "5560a026-a6c2-4140-ba7b-dd15fe692626", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02721, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 6.417524794000201, "model_name": "gpt-6-astra", "output_tokens": 51, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:57.704470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02721, "created_at": "2026-09-17T03:15:57.704Z" }, { "sequence": 1401, "attempt_id": "a36a5b39-b771-4953-8fb8-6a4419cd0f37", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006508, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.1632418850203976, "model_name": "claude-sonnet-5", "output_tokens": 89, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:58.007627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006508, "created_at": "2026-09-17T03:15:58.007Z" }, { "sequence": 1402, "attempt_id": "0eb511d5-a98c-4a57-be30-51cc04b6acf3", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.7741790609434247, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:58.672275+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:15:58.672Z" }, { "sequence": 1403, "attempt_id": "b434f51f-f68e-4ff1-8f37-06774423f657", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01547, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 3.03207984007895, "model_name": "claude-opus-5", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:58.749529+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01547, "created_at": "2026-09-17T03:15:58.749Z" }, { "sequence": 1404, "attempt_id": "434c7e36-6fee-49ed-afe7-808251ca3ccd", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.169032997917384, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:58.823933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T03:15:58.824Z" }, { "sequence": 1405, "attempt_id": "82507424-4cb0-4794-9224-cce493756a81", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005472, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.3850220440654084, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:59.467816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005472, "created_at": "2026-09-17T03:15:59.467Z" }, { "sequence": 1406, "attempt_id": "ba9d60a2-a0e2-4246-85a6-8b522d00f6cd", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.8089474890148267, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:00.353036+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000368, "created_at": "2026-09-17T03:16:00.353Z" }, { "sequence": 1407, "attempt_id": "a1715b63-f9a9-402b-b49c-fb609541dbd0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010732, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.9212906419998035, "model_name": "gpt-5.6-sol", "output_tokens": 59, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:00.701672+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010732, "created_at": "2026-09-17T03:16:00.701Z" }, { "sequence": 1408, "attempt_id": "99446fa0-7d9c-4c39-af8c-78421c997cff", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01729075, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 19.083074892056175, "model_name": "muse-spark-1.2", "output_tokens": 3324, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:02.123943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01729075, "created_at": "2026-09-17T03:16:02.124Z" }, { "sequence": 1409, "attempt_id": "aafd0f45-4a8a-4238-b5b4-825bf111446c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014594, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 9.843833927065134, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1014, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:02.954671+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014594, "created_at": "2026-09-17T03:16:02.954Z" }, { "sequence": 1410, "attempt_id": "59e6c1d2-ed51-4342-bd2d-21d529cb0bfd", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00482725, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 2.4445153640117496, "model_name": "muse-spark-1.2", "output_tokens": 392, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:03.220993+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00482725, "created_at": "2026-09-17T03:16:03.221Z" }, { "sequence": 1411, "attempt_id": "0eac881c-0f98-41be-a984-28d32c575856", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004585, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 3.31866834894754, "model_name": "muse-spark-1.3", "output_tokens": 335, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:03.747755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004585, "created_at": "2026-09-17T03:16:03.747Z" }, { "sequence": 1412, "attempt_id": "3da9a77b-b254-4b92-b1ef-97b79d27e7d3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 5.2820413479348645, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:04.031897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004944, "created_at": "2026-09-17T03:16:04.032Z" }, { "sequence": 1413, "attempt_id": "77869f0b-1ace-4cdd-b362-5420415b32b0", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00199125, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.323709764983505, "model_name": "gemini-3.8-flash", "output_tokens": 304, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:04.223328+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00199125, "created_at": "2026-09-17T03:16:04.223Z" }, { "sequence": 1414, "attempt_id": "c3177125-fb63-4ad2-865b-55078d7cb5b1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6092378080356866, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:04.433946+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T03:16:04.434Z" }, { "sequence": 1415, "attempt_id": "61db05f8-9189-4b96-8690-6eeec17984e5", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009086, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.959923560032621, "model_name": "gemini-3.1-pro-preview", "output_tokens": 568, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:04.857265+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009086, "created_at": "2026-09-17T03:16:04.857Z" }, { "sequence": 1416, "attempt_id": "a0ad403d-2a94-4e0c-a05e-b945fd56673c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.9543747609714046, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:05.262299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:16:05.262Z" }, { "sequence": 1417, "attempt_id": "b1dc0731-4b15-49cb-b499-33170961727b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.244080572971143, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:05.339174+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T03:16:05.340Z" }, { "sequence": 1418, "attempt_id": "fe235ce0-0a19-4d34-9054-07fbf4405c6a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.961956600076519, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:05.470955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:16:05.471Z" }, { "sequence": 1419, "attempt_id": "34637318-1f12-45af-a900-c16596a2323e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.513915000949055, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:05.714413+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T03:16:05.714Z" }, { "sequence": 1420, "attempt_id": "a4a56a35-fed6-4655-8dbf-1302173afbe7", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.6275190289597958, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:05.791489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:16:05.792Z" }, { "sequence": 1421, "attempt_id": "8bd8c240-45f5-4f4b-924b-f6b40b164200", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.1817935910075903, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:06.115298+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:16:06.115Z" }, { "sequence": 1422, "attempt_id": "4cdf795b-a5fd-48e4-a530-5c93c034f958", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.9254533520434052, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:06.223738+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:16:06.223Z" }, { "sequence": 1423, "attempt_id": "580b5c11-a3c5-4195-8d53-821867d9484f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.9173841370502487, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:06.464294+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:16:06.464Z" }, { "sequence": 1424, "attempt_id": "90a70edc-009c-436f-be26-c50d87cf3204", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003662, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 0.9933278320822865, "model_name": "muse-spark-1.2", "output_tokens": 124, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:06.862199+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003662, "created_at": "2026-09-17T03:16:06.862Z" }, { "sequence": 1425, "attempt_id": "d6b1b39d-baf8-4567-a8ae-6af058edb79e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.5172522939974442, "model_name": "gemini-3.8-flash", "output_tokens": 105, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:06.938971+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00122925, "created_at": "2026-09-17T03:16:06.939Z" }, { "sequence": 1426, "attempt_id": "c8d76b7b-d476-4d89-aba2-caf0e3f41e7f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001871, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.8302260030759498, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:07.769551+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001871, "created_at": "2026-09-17T03:16:07.769Z" }, { "sequence": 1427, "attempt_id": "72186336-0a7f-49ce-90e8-eca73b3ec003", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00344, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.0959997860481963, "model_name": "gemini-3.1-pro-preview", "output_tokens": 101, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:08.435495+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00344, "created_at": "2026-09-17T03:16:08.435Z" }, { "sequence": 1428, "attempt_id": "d2f4646d-f4ed-4adc-98bf-267550217eb3", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.098136610002257, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:08.638424+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005798, "created_at": "2026-09-17T03:16:08.638Z" }, { "sequence": 1429, "attempt_id": "deac0f84-f1be-4031-83ad-89f6345a0d65", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02609, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.538791846949607, "model_name": "gpt-6-astra", "output_tokens": 41, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:09.564738+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02609, "created_at": "2026-09-17T03:16:09.564Z" }, { "sequence": 1430, "attempt_id": "e987ea52-07e9-4a39-ac46-54c33db9706a", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01852, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.507633361965418, "model_name": "claude-opus-5", "output_tokens": 172, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:09.806392+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01852, "created_at": "2026-09-17T03:16:09.806Z" }, { "sequence": 1431, "attempt_id": "0c44a1bd-fdf7-4553-a6d7-8cf68913d024", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00438875, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 4.027216835995205, "model_name": "muse-spark-1.3", "output_tokens": 295, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:09.885912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00438875, "created_at": "2026-09-17T03:16:09.887Z" }, { "sequence": 1432, "attempt_id": "3ca87ebb-9486-4b66-abff-1fb6bf34b642", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005672, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.5688131119823083, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:10.081492+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005672, "created_at": "2026-09-17T03:16:10.081Z" }, { "sequence": 1433, "attempt_id": "04c21740-7bf7-4361-923b-9971d29f89ca", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005804, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.4438952569616958, "model_name": "gpt-5.6-luna", "output_tokens": 83, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:10.171602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005804, "created_at": "2026-09-17T03:16:10.173Z" }, { "sequence": 1434, "attempt_id": "4a047567-9f7a-4088-b6d6-e439240f505e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003615, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.0073461580323055, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:10.976386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003615, "created_at": "2026-09-17T03:16:10.976Z" }, { "sequence": 1435, "attempt_id": "97d59c38-b9ab-450f-b642-4227e02f6bf9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 3.3877902569947764, "model_name": "gpt-5.6-sol", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:11.235446+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011056, "created_at": "2026-09-17T03:16:11.235Z" }, { "sequence": 1436, "attempt_id": "0931c4dc-51c0-4de6-8a97-d100d79f05e1", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03881, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 6.554766725981608, "model_name": "claude-fable-5-1", "output_tokens": 207, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:12.745762+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03881, "created_at": "2026-09-17T03:16:12.745Z" }, { "sequence": 1437, "attempt_id": "3177f5e0-b95a-4964-a804-02222a084d3a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.3811056839767843, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:14.433046+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:16:14.433Z" }, { "sequence": 1438, "attempt_id": "3abf33f7-78d3-4084-ba6e-23e5ce904d0d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017195, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.3562364609679207, "model_name": "claude-opus-5", "output_tokens": 120, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:14.664074+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017195, "created_at": "2026-09-17T03:16:14.664Z" }, { "sequence": 1439, "attempt_id": "98586356-3609-4b23-a579-0ab3b937bfff", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006728, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.381715181050822, "model_name": "claude-sonnet-5", "output_tokens": 105, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:15.207651+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006728, "created_at": "2026-09-17T03:16:15.207Z" }, { "sequence": 1440, "attempt_id": "1617b36f-7bf7-4d35-8865-f2be591283ac", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.2166944530326873, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:15.728186+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:16:15.728Z" }, { "sequence": 1441, "attempt_id": "28823f8c-dcbd-40ef-b4a5-0dd6b987ddf4", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00478875, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 6.02493344899267, "model_name": "gemini-3.8-flash", "output_tokens": 1046, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:15.911169+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00478875, "created_at": "2026-09-17T03:16:15.911Z" }, { "sequence": 1442, "attempt_id": "d6811c2c-081a-47ec-931e-458f4b031919", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.8950124019756913, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:16.882724+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004968, "created_at": "2026-09-17T03:16:16.882Z" }, { "sequence": 1443, "attempt_id": "805e7fd4-5c23-4aa0-be7f-98fb3c2992d1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.0443938421085477, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:17.339281+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:16:17.339Z" }, { "sequence": 1444, "attempt_id": "f526f5cd-199f-45b4-8168-b824d098eb57", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.67257488402538, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:17.415616+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:16:17.415Z" }, { "sequence": 1445, "attempt_id": "b7ffbf56-456b-473d-94a3-fce7980d9df1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02144025, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 22.674961029086262, "model_name": "muse-spark-1.2", "output_tokens": 4278, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:17.509642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02144025, "created_at": "2026-09-17T03:16:17.509Z" }, { "sequence": 1446, "attempt_id": "a0f73be2-ac8a-4d09-8c8c-6bf1c0870ad1", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9204727009637281, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:18.410100+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:16:18.410Z" }, { "sequence": 1447, "attempt_id": "e1e91dcd-c74f-4ebd-8437-f9b4b9491488", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005856, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.76601202506572, "model_name": "gpt-5.6-terra", "output_tokens": 88, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:18.570770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005856, "created_at": "2026-09-17T03:16:18.570Z" }, { "sequence": 1448, "attempt_id": "d7ddded6-af5c-48a1-9ebf-78f64cc35f47", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015867, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 36.02220013702754, "model_name": "muse-spark-1.3", "output_tokens": 2989, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:18.911998+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015867, "created_at": "2026-09-17T03:16:18.912Z" }, { "sequence": 1449, "attempt_id": "07e08df9-af12-41bc-84a6-058483dbf8d4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014466, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 9.596139468019828, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1013, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:19.273410+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014466, "created_at": "2026-09-17T03:16:19.273Z" }, { "sequence": 1450, "attempt_id": "62a0c35b-66cf-4b27-9c75-3e59b326acc3", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0031575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.6764772731112316, "model_name": "gemini-3.8-flash", "output_tokens": 612, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:20.092416+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0031575, "created_at": "2026-09-17T03:16:20.092Z" }, { "sequence": 1451, "attempt_id": "71829c37-6963-49d7-bb79-c9f130d329f5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0127565, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 10.690787823987193, "model_name": "muse-spark-1.2", "output_tokens": 2253, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:20.942917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0127565, "created_at": "2026-09-17T03:16:20.943Z" }, { "sequence": 1452, "attempt_id": "84db5215-7fe7-4fdb-98a4-874c5ad47fd9", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.231770273996517, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:21.400400+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:16:21.400Z" }, { "sequence": 1453, "attempt_id": "a20fd5f3-c315-43b0-9d96-2530d55e0e69", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006668, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.42094891902525, "model_name": "claude-sonnet-5", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:21.771565+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006668, "created_at": "2026-09-17T03:16:21.771Z" }, { "sequence": 1454, "attempt_id": "2addb9b9-4d91-4bd2-8b8b-e8fe38c65ba5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01667, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.8696905840188265, "model_name": "claude-opus-5", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:21.861210+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01667, "created_at": "2026-09-17T03:16:21.861Z" }, { "sequence": 1455, "attempt_id": "b2fc2f8d-dc50-4d6f-b810-bf000ce30e7e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.8218922490486875, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:22.467204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:16:22.467Z" }, { "sequence": 1456, "attempt_id": "dbe0c4fa-df67-4d82-b954-59494605b8e6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.738374840002507, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:22.757934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:16:22.758Z" }, { "sequence": 1457, "attempt_id": "a55a791b-b65e-42bc-bad4-360cb0d4ec9c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007976, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 6.363318598945625, "model_name": "gemini-3.1-pro-preview", "output_tokens": 473, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:23.322815+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007976, "created_at": "2026-09-17T03:16:23.322Z" }, { "sequence": 1458, "attempt_id": "f0c758b4-09a6-48cc-a739-3c9b70673889", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00553925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.068970128078945, "model_name": "muse-spark-1.3", "output_tokens": 556, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:23.651851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00553925, "created_at": "2026-09-17T03:16:23.651Z" }, { "sequence": 1459, "attempt_id": "b23dcdf7-f188-4b53-9bd8-f8f6790669f8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000594, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.8817143809283152, "model_name": "gpt-5.6-luna", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:23.818895+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000594, "created_at": "2026-09-17T03:16:23.819Z" }, { "sequence": 1460, "attempt_id": "f2666da0-d971-4a6f-ab55-c57920327470", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011684, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.4554199189879, "model_name": "gpt-5.6-sol", "output_tokens": 91, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:23.932739+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011684, "created_at": "2026-09-17T03:16:23.932Z" }, { "sequence": 1461, "attempt_id": "1c9ce7c6-09b4-4e2b-892e-a4fd64f3fd55", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00758775, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.458426292985678, "model_name": "muse-spark-1.2", "output_tokens": 1038, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:24.011089+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00758775, "created_at": "2026-09-17T03:16:24.011Z" }, { "sequence": 1462, "attempt_id": "2d541e28-52bf-412a-b307-2f7cdc07cf0f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005748, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.2548505270387977, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:24.116370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005748, "created_at": "2026-09-17T03:16:24.116Z" }, { "sequence": 1463, "attempt_id": "bdaa41a9-dd1e-43ae-b701-f70b1b6ee6ea", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.0388778370106593, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:24.438099+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003789, "created_at": "2026-09-17T03:16:24.438Z" }, { "sequence": 1464, "attempt_id": "6fd999fd-1f57-4045-aa34-ce685da1c1b3", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6931344530312344, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:25.208418+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:16:25.208Z" }, { "sequence": 1465, "attempt_id": "d2d1b947-597e-47f2-bc8c-087e005846e5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0094755, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 15.39462828997057, "model_name": "muse-spark-1.3", "output_tokens": 1481, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:25.566527+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0094755, "created_at": "2026-09-17T03:16:25.566Z" }, { "sequence": 1466, "attempt_id": "adeb9dbd-52cc-4a9e-867a-4b930bb4f9f9", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0019785, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.8202079840702936, "model_name": "gemini-3.8-flash", "output_tokens": 285, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:25.655163+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0019785, "created_at": "2026-09-17T03:16:25.655Z" }, { "sequence": 1467, "attempt_id": "21a5b566-5e46-4403-a351-7d52553ab6fc", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.203929394017905, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:26.846839+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:16:26.847Z" }, { "sequence": 1468, "attempt_id": "fdff8d66-0114-4f56-a7f0-8df04f55eb9b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7253484439570457, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:27.016201+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:16:27.016Z" }, { "sequence": 1469, "attempt_id": "47a73071-0c4e-4d72-832f-ff4fdfe16441", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016045, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.9327957310015336, "model_name": "claude-opus-5", "output_tokens": 74, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:27.107597+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016045, "created_at": "2026-09-17T03:16:27.107Z" }, { "sequence": 1470, "attempt_id": "0c4b825a-e894-4d7c-ba90-c43dd38b447a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007288, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.4899997679749504, "model_name": "claude-sonnet-5", "output_tokens": 161, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:27.686949+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007288, "created_at": "2026-09-17T03:16:27.687Z" }, { "sequence": 1471, "attempt_id": "a16a411a-b5fb-4632-a271-6af9fff5744c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005952, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.0367783510591835, "model_name": "gpt-5.6-terra", "output_tokens": 96, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:27.767682+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005952, "created_at": "2026-09-17T03:16:27.767Z" }, { "sequence": 1472, "attempt_id": "5d41fca7-7731-48ee-a733-9670ab00a3c0", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008642, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.9525555149884894, "model_name": "gemini-3.1-pro-preview", "output_tokens": 518, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:28.495715+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008642, "created_at": "2026-09-17T03:16:28.495Z" }, { "sequence": 1473, "attempt_id": "5f6089bf-2872-4c7b-a49e-9eeb738804e5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9611291820183396, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:28.892492+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004968, "created_at": "2026-09-17T03:16:28.892Z" }, { "sequence": 1474, "attempt_id": "10a69016-d841-465e-a147-29f8b9ba2ceb", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.8213758430210873, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:29.589411+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:16:29.589Z" }, { "sequence": 1475, "attempt_id": "7e13ac60-39af-4754-a418-17006ea9fc63", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003944, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.206380732008256, "model_name": "gemini-3.1-pro-preview", "output_tokens": 137, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:30.314317+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003944, "created_at": "2026-09-17T03:16:30.314Z" }, { "sequence": 1476, "attempt_id": "7b5960e3-6436-4dee-895c-835009c330cf", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008329, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 6.723455403931439, "model_name": "muse-spark-1.2", "output_tokens": 1193, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:30.621290+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008329, "created_at": "2026-09-17T03:16:30.621Z" }, { "sequence": 1477, "attempt_id": "d8defde0-2e73-42a1-be1e-fb49ee7fa45c", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0021375, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.6981255878927186, "model_name": "gemini-3.8-flash", "output_tokens": 340, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:30.884522+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0021375, "created_at": "2026-09-17T03:16:30.884Z" }, { "sequence": 1478, "attempt_id": "86ee5c69-6ce3-49ca-9def-6f3f899b88f5", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.622902411967516, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:31.327915+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:16:31.328Z" }, { "sequence": 1479, "attempt_id": "51809868-df0f-4d83-a3cf-ab45ec2b1895", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01868625, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 37.17285589396488, "model_name": "muse-spark-1.3", "output_tokens": 3630, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:31.732676+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01868625, "created_at": "2026-09-17T03:16:31.732Z" }, { "sequence": 1480, "attempt_id": "1ff5f35a-caa0-4224-809d-442781370eec", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.2259565429994836, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:32.196661+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:16:32.196Z" }, { "sequence": 1481, "attempt_id": "5d3ccc1a-ce12-4281-a670-ae1350d94ad3", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8819109620526433, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:32.277621+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:16:32.277Z" }, { "sequence": 1482, "attempt_id": "31b2930c-ebbb-477f-adbc-8813fa405877", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.6399153870297596, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:32.359716+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:16:32.361Z" }, { "sequence": 1483, "attempt_id": "787f21f8-b673-4b83-b307-cd867a049468", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2981639290228486, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:32.705152+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:16:32.705Z" }, { "sequence": 1484, "attempt_id": "5297534a-6eba-4b82-80af-ad6d6859d0c1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9285812089219689, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:32.795684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:16:32.797Z" }, { "sequence": 1485, "attempt_id": "3cbd17ae-d9e5-4262-8733-48aab77c3773", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.0042898550163954, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:32.967708+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:16:32.967Z" }, { "sequence": 1486, "attempt_id": "b66d1ee9-dd0c-4d5c-82d2-7ba1f14536d3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9596412770915776, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:33.237578+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:16:33.237Z" }, { "sequence": 1487, "attempt_id": "50066a40-2c1f-4184-86b9-6d6df7bf35f7", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.821519845048897, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:33.617517+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:16:33.617Z" }, { "sequence": 1488, "attempt_id": "9ce33f96-b0ed-4385-a211-8b2ce3529f45", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005501, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.847028578980826, "model_name": "muse-spark-1.3", "output_tokens": 547, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:33.696518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005501, "created_at": "2026-09-17T03:16:33.696Z" }, { "sequence": 1489, "attempt_id": "c3b96431-469a-4b34-8d3c-4aff4f6942c9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007105, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.045810213079676, "model_name": "muse-spark-1.3", "output_tokens": 905, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:33.773381+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007105, "created_at": "2026-09-17T03:16:33.773Z" }, { "sequence": 1490, "attempt_id": "7c68d7c5-becc-499d-88d6-d9dedb34b7ee", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00120525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.5154273069929332, "model_name": "gemini-3.8-flash", "output_tokens": 98, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:33.955495+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00120525, "created_at": "2026-09-17T03:16:33.955Z" }, { "sequence": 1491, "attempt_id": "92f1a4b6-28c4-4671-8fa2-ed291142627a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0057305, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.513560279039666, "model_name": "muse-spark-1.2", "output_tokens": 601, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:34.087067+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0057305, "created_at": "2026-09-17T03:16:34.087Z" }, { "sequence": 1492, "attempt_id": "cb05bb9f-cf53-4793-8a0d-493e167359d1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00405675, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.6657490889774635, "model_name": "muse-spark-1.2", "output_tokens": 216, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:34.711777+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00405675, "created_at": "2026-09-17T03:16:34.711Z" }, { "sequence": 1493, "attempt_id": "965a6614-92b1-493b-9c3c-3bcfc28918c6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.0590460241073743, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:34.909059+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:16:34.909Z" }, { "sequence": 1494, "attempt_id": "765d35bf-2def-4f31-8013-94aaf4601135", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003866, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.0638955559115857, "model_name": "gemini-3.1-pro-preview", "output_tokens": 136, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:35.423981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003866, "created_at": "2026-09-17T03:16:35.424Z" }, { "sequence": 1495, "attempt_id": "90f9c8c1-1096-4083-8064-6077638d89c3", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.5733935890020803, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:35.607358+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:16:35.607Z" }, { "sequence": 1496, "attempt_id": "9f1abcff-3e1a-437f-a78e-cd38d8660318", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004333, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.832102475920692, "model_name": "muse-spark-1.3", "output_tokens": 281, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:35.705106+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004333, "created_at": "2026-09-17T03:16:35.705Z" }, { "sequence": 1497, "attempt_id": "028a3135-096c-4d5f-bee0-6d866dce2798", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.745292912935838, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:35.909426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:16:35.909Z" }, { "sequence": 1498, "attempt_id": "dca62530-4a35-4de4-90e2-3f919bda2daf", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006538, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.4513864050386474, "model_name": "claude-sonnet-5", "output_tokens": 86, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:36.225073+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006538, "created_at": "2026-09-17T03:16:36.225Z" }, { "sequence": 1499, "attempt_id": "c9b6135c-0c89-41fd-bc69-fa991ef1c3da", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005556, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7237675728974864, "model_name": "gpt-5.6-terra", "output_tokens": 63, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:36.513006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005556, "created_at": "2026-09-17T03:16:36.513Z" }, { "sequence": 1500, "attempt_id": "a32c2fa9-9528-4998-85a6-86e285aff548", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.2739437649725005, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:36.592954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:16:36.593Z" }, { "sequence": 1501, "attempt_id": "81e688f9-a276-4b59-a57e-d9ae9196d025", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 12.781366653973237, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:36.793326+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:16:36.793Z" }, { "sequence": 1502, "attempt_id": "dea76067-5eb1-4d8f-b6bc-8d5d1e5e3fe6", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.2062135859159753, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:36.990636+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:16:36.990Z" }, { "sequence": 1503, "attempt_id": "c9f0dabd-f162-4b8b-a4bb-3f9b640f6543", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01662, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.432309611933306, "model_name": "claude-opus-5", "output_tokens": 97, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:37.129144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01662, "created_at": "2026-09-17T03:16:37.129Z" }, { "sequence": 1504, "attempt_id": "85120673-a7a7-4f22-9cb5-a65137c42030", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00057, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.217145559960045, "model_name": "gpt-5.6-luna", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:37.210544+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00057, "created_at": "2026-09-17T03:16:37.210Z" }, { "sequence": 1505, "attempt_id": "1af0aee2-ce35-44c4-8a83-6e467dc72984", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 1.0003709819866344, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:38.070397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001833, "created_at": "2026-09-17T03:16:38.070Z" }, { "sequence": 1506, "attempt_id": "0c0169ec-3104-4498-bf1f-98f465897541", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.0537814479321241, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:38.343987+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009752, "created_at": "2026-09-17T03:16:38.344Z" }, { "sequence": 1507, "attempt_id": "f11fa3bd-690c-43f9-b2c8-d7f0dfc8c539", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.9293416609289125, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:38.600671+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01421, "created_at": "2026-09-17T03:16:38.600Z" }, { "sequence": 1508, "attempt_id": "0d2da944-bb8e-46a2-b1dd-6a0c76621f74", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.8654006279539317, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:38.737788+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005684, "created_at": "2026-09-17T03:16:38.737Z" }, { "sequence": 1509, "attempt_id": "c667cec2-e2d1-497e-aa96-b247d96c7bc7", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.6661378790158778, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:38.876966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02438, "created_at": "2026-09-17T03:16:38.877Z" }, { "sequence": 1510, "attempt_id": "b43591da-e3b6-401c-8e28-898e27aae36f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.0813770699314773, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:39.231132+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004904, "created_at": "2026-09-17T03:16:39.231Z" }, { "sequence": 1511, "attempt_id": "057a6b5c-0a32-440d-a9f1-f9af3e056896", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 3.2990403630537912, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:39.892340+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02844, "created_at": "2026-09-17T03:16:39.892Z" }, { "sequence": 1512, "attempt_id": "1ce87fa5-9ffd-4815-ac06-e7d7f36501cf", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0251946720527485, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:39.983479+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:16:39.983Z" }, { "sequence": 1513, "attempt_id": "24971451-f8c3-4f0e-bd0d-283a176b68dd", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001155, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.6964867749484256, "model_name": "gemini-3.8-flash", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:40.511311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001155, "created_at": "2026-09-17T03:16:40.511Z" }, { "sequence": 1514, "attempt_id": "37ed2969-379a-47bf-baff-0e848c644751", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003585, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.922548152972013, "model_name": "gemini-3.8-flash", "output_tokens": 726, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:40.609936+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003585, "created_at": "2026-09-17T03:16:40.610Z" }, { "sequence": 1515, "attempt_id": "1ef293c1-de34-423f-bca5-0f721e73f57f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007064, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 5.429854942951351, "model_name": "gemini-3.1-pro-preview", "output_tokens": 397, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:40.934602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007064, "created_at": "2026-09-17T03:16:40.934Z" }, { "sequence": 1516, "attempt_id": "8e1c04be-1eac-4a42-9ced-b39d5c2c6b34", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003166, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.9893517460441217, "model_name": "gemini-3.1-pro-preview", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:41.668343+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003166, "created_at": "2026-09-17T03:16:41.668Z" }, { "sequence": 1517, "attempt_id": "cf61abec-1591-4ee2-84f2-b17bea5acc9d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074645, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.3754498270573094, "model_name": "muse-spark-1.2", "output_tokens": 1009, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:41.749231+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074645, "created_at": "2026-09-17T03:16:41.749Z" }, { "sequence": 1518, "attempt_id": "d1f46fae-98c5-464b-adaf-4c900d158144", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0042455, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 2.5206191450124606, "model_name": "muse-spark-1.3", "output_tokens": 261, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:41.831847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0042455, "created_at": "2026-09-17T03:16:41.831Z" }, { "sequence": 1519, "attempt_id": "88468193-8a29-4369-a77d-b11099661343", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00366325, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 2.2098521419102326, "model_name": "muse-spark-1.2", "output_tokens": 124, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:42.193689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00366325, "created_at": "2026-09-17T03:16:42.193Z" }, { "sequence": 1520, "attempt_id": "9fd6a58c-d333-4a6d-abb1-8bf357a7bcb0", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 1.497398411971517, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:42.510648+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001884, "created_at": "2026-09-17T03:16:42.510Z" }, { "sequence": 1521, "attempt_id": "dee6f1b4-f777-4562-a596-1406a69c38dd", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006916, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 2.5784367159940302, "model_name": "claude-sonnet-5", "output_tokens": 118, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:43.264221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006916, "created_at": "2026-09-17T03:16:43.264Z" }, { "sequence": 1522, "attempt_id": "8276108b-2ab5-4c5f-a4db-3e378c60fb4a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005752, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.8353930540615693, "model_name": "gpt-5.6-terra", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:43.746888+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005752, "created_at": "2026-09-17T03:16:43.747Z" }, { "sequence": 1523, "attempt_id": "227d4ec0-ef4f-461b-908c-006593b95c73", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011236, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.434924514964223, "model_name": "gpt-5.6-sol", "output_tokens": 79, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:44.266858+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011236, "created_at": "2026-09-17T03:16:44.267Z" }, { "sequence": 1524, "attempt_id": "11641791-ae31-4080-892e-a80c73d9f383", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.6181842000223696, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:44.367770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02484, "created_at": "2026-09-17T03:16:44.367Z" }, { "sequence": 1525, "attempt_id": "d8365643-d605-47a9-b0a2-9a2175a2571b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02955, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 4.385055043036118, "model_name": "claude-fable-5-1", "output_tokens": 17, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:44.448886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02955, "created_at": "2026-09-17T03:16:44.449Z" }, { "sequence": 1526, "attempt_id": "83593da5-1a8d-4ed6-88ab-aa6304128c9c", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.9148807669989765, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:44.742342+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003761, "created_at": "2026-09-17T03:16:44.742Z" }, { "sequence": 1527, "attempt_id": "92deab3c-f1d7-4453-a913-04cbc6f8a168", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01759, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 4.2332485150545835, "model_name": "claude-opus-5", "output_tokens": 130, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:44.834559+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01759, "created_at": "2026-09-17T03:16:44.834Z" }, { "sequence": 1528, "attempt_id": "3a588645-a3dc-4577-abfb-5add907673e1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 8.174780450062826, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:46.597488+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004904, "created_at": "2026-09-17T03:16:46.597Z" }, { "sequence": 1529, "attempt_id": "ea95177d-35d6-4439-ab09-a7ff91a9ad7f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004059, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 3.2973644790472463, "model_name": "gemini-3.8-flash", "output_tokens": 850, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:46.680665+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004059, "created_at": "2026-09-17T03:16:46.682Z" }, { "sequence": 1530, "attempt_id": "75474af1-eb69-4da2-905e-752e072d636b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076005, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 10.676851279102266, "model_name": "muse-spark-1.3", "output_tokens": 1041, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:46.761648+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0076005, "created_at": "2026-09-17T03:16:46.761Z" }, { "sequence": 1531, "attempt_id": "c6d1cb59-5579-48c5-b6fb-17e88354c25a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006788, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.7119107090402395, "model_name": "claude-sonnet-5", "output_tokens": 111, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:47.624454+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006788, "created_at": "2026-09-17T03:16:47.624Z" }, { "sequence": 1532, "attempt_id": "11d9618e-e14c-4cf4-b372-4210d14c2590", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.0267259620595723, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:47.707661+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:16:47.707Z" }, { "sequence": 1533, "attempt_id": "c771400a-25ed-4114-a1b0-513e9cb2f5b5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016095, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.9417102159932256, "model_name": "claude-opus-5", "output_tokens": 76, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:47.786564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016095, "created_at": "2026-09-17T03:16:47.786Z" }, { "sequence": 1534, "attempt_id": "8fef2b49-f3a5-4295-9274-a8a9978758ae", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005082, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 4.326736490940675, "model_name": "muse-spark-1.3", "output_tokens": 444, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:48.675372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005082, "created_at": "2026-09-17T03:16:48.675Z" }, { "sequence": 1535, "attempt_id": "01386172-d8da-4758-b792-648dd2e1263a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9697817480191588, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:48.813841+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:16:48.813Z" }, { "sequence": 1536, "attempt_id": "581d9d3a-d3ad-475a-b686-7434f662f2c1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.113491602940485, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:48.960027+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:16:48.960Z" }, { "sequence": 1537, "attempt_id": "d8d1d325-eef1-4f1f-860e-351cfcf59fdb", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 4.813780969008803, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:49.340661+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:16:49.340Z" }, { "sequence": 1538, "attempt_id": "8b183878-7c44-4841-b58b-94dd49862cef", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005724, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.65151071397122, "model_name": "gpt-5.6-terra", "output_tokens": 77, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:49.422691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005724, "created_at": "2026-09-17T03:16:49.422Z" }, { "sequence": 1539, "attempt_id": "d307335e-f23c-422f-9f71-b9ddaf5b56b5", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010844, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 6.878681114059873, "model_name": "gemini-3.1-pro-preview", "output_tokens": 710, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:49.502195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010844, "created_at": "2026-09-17T03:16:49.503Z" }, { "sequence": 1540, "attempt_id": "4489c1cc-e410-4c8f-8f42-947a5bb9fcc3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000588, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.8109861790435389, "model_name": "gpt-5.6-luna", "output_tokens": 90, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:49.597828+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000588, "created_at": "2026-09-17T03:16:49.597Z" }, { "sequence": 1541, "attempt_id": "c933414b-7f4e-4b23-81a8-da6fff8e3e66", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9019740830408409, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:49.794784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:16:49.794Z" }, { "sequence": 1542, "attempt_id": "e4d3f94a-817a-4753-bab1-6713f3adb378", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005727999999999999, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 7.959214351023547, "model_name": "gpt-5.6-luna", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:50.233339+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005727999999999999, "created_at": "2026-09-17T03:16:50.233Z" }, { "sequence": 1543, "attempt_id": "4978f53c-a3de-408f-b248-3b8862a4de1e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0019125, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.6762777790427208, "model_name": "gemini-3.8-flash", "output_tokens": 280, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:50.432170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0019125, "created_at": "2026-09-17T03:16:50.432Z" }, { "sequence": 1544, "attempt_id": "ed07c130-647e-4287-bd77-bd0bf7c39954", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006187, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 5.9965426289709285, "model_name": "muse-spark-1.2", "output_tokens": 704, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:50.513667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006187, "created_at": "2026-09-17T03:16:50.513Z" }, { "sequence": 1545, "attempt_id": "b3e01915-6b38-4acb-baf7-915af95bdbe0", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.028883810970001, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:50.903132+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:16:50.903Z" }, { "sequence": 1546, "attempt_id": "b347d70e-d024-4f91-a536-e20cf1c3f836", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9335799060063437, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:51.619276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:16:51.619Z" }, { "sequence": 1547, "attempt_id": "8abf90f5-a2e3-4ee6-8381-b3b85949f5e2", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.0427591720363125, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:51.701683+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:16:51.701Z" }, { "sequence": 1548, "attempt_id": "c597b3dd-39af-4180-8362-6e8ee9c1fb89", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.3018937619635835, "model_name": "gpt-5.6-terra", "output_tokens": 30, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:51.893195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0051, "created_at": "2026-09-17T03:16:51.893Z" }, { "sequence": 1549, "attempt_id": "7abb9c1d-9992-4854-981b-0cd294561fdf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.19415931601543, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:52.708181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:16:52.708Z" }, { "sequence": 1550, "attempt_id": "f831bd8d-b32b-41b6-a94b-5343d8f1a310", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7318052790360525, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:52.795810+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:16:52.796Z" }, { "sequence": 1551, "attempt_id": "68ce1d65-1184-4556-b4d5-b888f4f6c66c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.385378777049482, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:52.887843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:16:52.887Z" }, { "sequence": 1552, "attempt_id": "704d5592-52d8-4230-8c6c-1dbb808d70b8", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.9915095849428326, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:52.970116+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:16:52.970Z" }, { "sequence": 1553, "attempt_id": "8406fa47-681a-4755-85fa-4ceef25b9b72", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00587075, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 3.629867520998232, "model_name": "muse-spark-1.2", "output_tokens": 634, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:53.055914+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00587075, "created_at": "2026-09-17T03:16:53.056Z" }, { "sequence": 1554, "attempt_id": "b3f270d9-70a3-4f74-b8f5-b04e9b81477f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001239, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.4983062079409137, "model_name": "gemini-3.8-flash", "output_tokens": 107, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:53.286024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001239, "created_at": "2026-09-17T03:16:53.286Z" }, { "sequence": 1555, "attempt_id": "87f96299-ffaf-485d-b357-801067059ab5", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008072, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 5.864690772956237, "model_name": "gemini-3.1-pro-preview", "output_tokens": 481, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:16:53.731633+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008072, "created_at": "2026-09-17T03:16:53.731Z" }, { "sequence": 1556, "attempt_id": "9ad60d0a-6eaf-4486-863e-58d8d97af445", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.5027740070363507, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:53.818191+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:16:53.818Z" }, { "sequence": 1557, "attempt_id": "943162b9-9aee-4757-83e6-e7598dcd6e68", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6952014289563522, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:54.074783+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T03:16:54.074Z" }, { "sequence": 1558, "attempt_id": "056a57fe-4194-470d-ba78-2205090bab4f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00323, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.594518308993429, "model_name": "gemini-3.1-pro-preview", "output_tokens": 83, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:54.296531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00323, "created_at": "2026-09-17T03:16:54.296Z" }, { "sequence": 1559, "attempt_id": "234b4930-73a1-47e9-a7e8-18ec8d894948", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004146, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.7720898890402168, "model_name": "muse-spark-1.2", "output_tokens": 237, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:54.651866+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004146, "created_at": "2026-09-17T03:16:54.652Z" }, { "sequence": 1560, "attempt_id": "57b43732-e96b-4a13-bfdf-61ce1f8be619", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00381025, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.8686451449757442, "model_name": "muse-spark-1.3", "output_tokens": 158, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:54.735139+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00381025, "created_at": "2026-09-17T03:16:54.735Z" }, { "sequence": 1561, "attempt_id": "9633a2f4-f395-4e29-8a79-49c58dfa9b74", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005346, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1157187060452998, "model_name": "gpt-5.6-terra", "output_tokens": 35, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:55.274461+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005346, "created_at": "2026-09-17T03:16:55.274Z" }, { "sequence": 1562, "attempt_id": "2b6b019d-5fc4-4cc8-8b9f-4416cc0badab", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.4016748709836975, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:55.357594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:16:55.357Z" }, { "sequence": 1563, "attempt_id": "a7bacbb7-e9e4-4882-b91a-d514e2a7957c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016475, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.6135436730692163, "model_name": "claude-opus-5", "output_tokens": 69, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:55.669798+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016475, "created_at": "2026-09-17T03:16:55.669Z" }, { "sequence": 1564, "attempt_id": "d1af9cfe-1f2a-49e9-9b22-5000fc7c69d7", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.0095601440407336, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:55.825574+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:16:55.825Z" }, { "sequence": 1565, "attempt_id": "6d4fcfac-c2e4-4efe-8e34-be415deb2815", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005766, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.6917035200167447, "model_name": "gpt-5.6-luna", "output_tokens": 70, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:56.072406+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005766, "created_at": "2026-09-17T03:16:56.072Z" }, { "sequence": 1566, "attempt_id": "8595e099-63a2-4979-9ee1-2ae4f6cd5975", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00139125, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.3815137699712068, "model_name": "gemini-3.8-flash", "output_tokens": 129, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:56.196404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00139125, "created_at": "2026-09-17T03:16:56.196Z" }, { "sequence": 1567, "attempt_id": "6550f0fd-b7a2-432d-9db2-44a1046dbf8b", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000378, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.1231810949975625, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:56.481157+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000378, "created_at": "2026-09-17T03:16:56.481Z" }, { "sequence": 1568, "attempt_id": "f9afbaf3-cfa7-4e96-bc85-380abf481597", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.48488995898515, "model_name": "claude-sonnet-5", "output_tokens": 110, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:56.623216+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007, "created_at": "2026-09-17T03:16:56.623Z" }, { "sequence": 1569, "attempt_id": "92e52fae-5e84-448d-87b2-8453b7c93ae5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00669525, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 8.698116454994306, "model_name": "muse-spark-1.3", "output_tokens": 828, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:57.736835+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00669525, "created_at": "2026-09-17T03:16:57.737Z" }, { "sequence": 1570, "attempt_id": "9bb50283-ae5c-447e-bc51-f2f54c88ac3f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.1820595170138404, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:57.835291+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:16:57.835Z" }, { "sequence": 1571, "attempt_id": "8c0ff8cd-fad9-42e8-ad75-f91780178f6e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8538779829395935, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:58.008955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:16:58.009Z" }, { "sequence": 1572, "attempt_id": "72310f8c-5899-43ae-a648-6e44d7bb13a4", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 5.196818795055151, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:58.167253+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:16:58.167Z" }, { "sequence": 1573, "attempt_id": "e88f6983-d51a-4e9a-94c5-c092644f4fe3", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9203671929426491, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:58.250032+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:16:58.250Z" }, { "sequence": 1574, "attempt_id": "dffd9a69-984c-4bcc-bef0-f56ea3e7ad5a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00510925, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.894737556925975, "model_name": "muse-spark-1.2", "output_tokens": 436, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:58.646978+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00510925, "created_at": "2026-09-17T03:16:58.647Z" }, { "sequence": 1575, "attempt_id": "a3c74ce6-bfb5-498e-ac12-0dab052bc428", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004816, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 3.3717171429889277, "model_name": "muse-spark-1.3", "output_tokens": 367, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:58.809119+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004816, "created_at": "2026-09-17T03:16:58.809Z" }, { "sequence": 1576, "attempt_id": "160d9c92-51de-4b26-a1d8-f6715570da36", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.106411244953051, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:58.890384+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:16:58.890Z" }, { "sequence": 1577, "attempt_id": "cb96cc91-2151-408c-ba20-590ecd0f0114", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.090792002971284, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:59.006681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:16:59.006Z" }, { "sequence": 1578, "attempt_id": "1f65cd56-ecad-4888-8450-1664dda0086f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0157816070131958, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:59.108495+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:16:59.108Z" }, { "sequence": 1579, "attempt_id": "e850aa5f-3c51-433a-94f2-420d68783f8c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.497054898063652, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:59.404242+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:16:59.404Z" }, { "sequence": 1580, "attempt_id": "19d5730c-387d-48c8-a7ba-7fb9de404d8c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.6493610350880772, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:59.488699+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:16:59.488Z" }, { "sequence": 1581, "attempt_id": "fc135f69-7a3f-4b43-be61-5ebceefb2e57", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.9890992300352082, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:59.718973+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00035009999999999994, "created_at": "2026-09-17T03:16:59.719Z" }, { "sequence": 1582, "attempt_id": "7680fb1f-ba31-4876-9db7-0fe2e8e7b69e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00138525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.6503496440127492, "model_name": "gemini-3.8-flash", "output_tokens": 146, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:59.981853+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00138525, "created_at": "2026-09-17T03:16:59.982Z" }, { "sequence": 1583, "attempt_id": "6907efcc-a052-4faf-b58a-e74c628292a8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003755, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.0576571830315515, "model_name": "muse-spark-1.2", "output_tokens": 145, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:00.078295+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003755, "created_at": "2026-09-17T03:17:00.079Z" }, { "sequence": 1584, "attempt_id": "8222c5cc-22cb-4ed7-8155-6a04f42622fc", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00758, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 5.824380330042914, "model_name": "gemini-3.1-pro-preview", "output_tokens": 430, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:00.559900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00758, "created_at": "2026-09-17T03:17:00.560Z" }, { "sequence": 1585, "attempt_id": "169ee1cc-7c86-46ce-a59c-f81b4a728720", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.2700655579101294, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:17:00.842048+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:17:00.842Z" }, { "sequence": 1586, "attempt_id": "0dcdc0fd-ea51-4e41-a04c-e1a0de9a15c7", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003482, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.672649472951889, "model_name": "gemini-3.1-pro-preview", "output_tokens": 104, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:00.925722+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003482, "created_at": "2026-09-17T03:17:00.926Z" }, { "sequence": 1587, "attempt_id": "c10bd7c8-e0d8-4987-9b63-079b246f2913", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00418425, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.6001617120346054, "model_name": "muse-spark-1.3", "output_tokens": 246, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:01.490886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00418425, "created_at": "2026-09-17T03:17:01.491Z" }, { "sequence": 1588, "attempt_id": "73b763c4-8399-499a-b2dd-1b39ca4dacce", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005832, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7159525220049545, "model_name": "gpt-5.6-terra", "output_tokens": 75, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:01.877954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005832, "created_at": "2026-09-17T03:17:01.878Z" }, { "sequence": 1589, "attempt_id": "aa435b29-22de-4fc5-8006-1aeed601fc73", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005916000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.4661026389803737, "model_name": "gpt-5.6-luna", "output_tokens": 82, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:02.113411+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005916000000000001, "created_at": "2026-09-17T03:17:02.113Z" }, { "sequence": 1590, "attempt_id": "9439322a-9611-4345-9734-a57bcf97d196", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011264, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.067560314084403, "model_name": "gpt-5.6-sol", "output_tokens": 70, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:02.198755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011264, "created_at": "2026-09-17T03:17:02.200Z" }, { "sequence": 1591, "attempt_id": "b1f2c767-b78c-4ee4-8254-7da1f298de65", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-019", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.6990466930437833, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:17:02.279990+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:17:02.281Z" }, { "sequence": 1592, "attempt_id": "1c32b50d-e41b-45df-b620-223b2f993bb2", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01817, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.449233538005501, "model_name": "claude-opus-5", "output_tokens": 140, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:02.639165+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01817, "created_at": "2026-09-17T03:17:02.639Z" }, { "sequence": 1593, "attempt_id": "2d1154ed-81ba-4b86-b3ce-70546202ebac", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008108, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.748614507028833, "model_name": "claude-sonnet-5", "output_tokens": 224, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:03.237645+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008108, "created_at": "2026-09-17T03:17:03.237Z" }, { "sequence": 1594, "attempt_id": "6b909df4-dae6-4b22-9192-7804c0d92879", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02691, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.5810546409338713, "model_name": "gpt-6-astra", "output_tokens": 45, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:03.385520+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02691, "created_at": "2026-09-17T03:17:03.385Z" }, { "sequence": 1595, "attempt_id": "764044fd-41d9-41ed-ba15-5822c0bd5a97", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00306975, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.458273339900188, "model_name": "gemini-3.8-flash", "output_tokens": 576, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:17:03.469193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00306975, "created_at": "2026-09-17T03:17:03.470Z" }, { "sequence": 1596, "attempt_id": "45a53255-7902-4839-9193-ab8431303e99", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03056, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.9368084309389815, "model_name": "claude-fable-5-1", "output_tokens": 24, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:17:04.027108+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03056, "created_at": "2026-09-17T03:17:04.027Z" }, { "sequence": 1597, "attempt_id": "b4b12539-f76c-455d-80dc-a9e78fbd2110", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.012361787026748, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:04.332976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:17:04.333Z" }, { "sequence": 1598, "attempt_id": "ff914b08-6d51-429f-8fd0-2104e92e332d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.052003897028044, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:04.416981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:17:04.417Z" }, { "sequence": 1599, "attempt_id": "6dbe860b-4d38-42ac-9238-87b30b0c9cc4", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.738166572060436, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:04.499928+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:17:04.500Z" }, { "sequence": 1600, "attempt_id": "6d3a49c6-b2a5-4f0d-9e18-fce584d8db0f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3206220470601693, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:04.883980+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:17:04.884Z" }, { "sequence": 1601, "attempt_id": "a7f3c027-7843-456f-843e-6dfaf39f9e90", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1851296430686489, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:05.295803+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:17:05.295Z" }, { "sequence": 1602, "attempt_id": "9539ea20-15da-4b23-b075-f0414b5d5708", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.109672823920846, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:05.527023+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:17:05.527Z" }, { "sequence": 1603, "attempt_id": "810a1b26-03c2-4cdb-b8d7-fd2803f32ded", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.2287813019938767, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:05.698308+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:17:05.698Z" }, { "sequence": 1604, "attempt_id": "579777dc-030e-49ce-8af2-4bd366427614", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.5377004919573665, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:05.817996+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:17:05.818Z" }, { "sequence": 1605, "attempt_id": "8dd64f69-35f2-4a41-b7c9-0b131da27d44", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8969985359581187, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:05.903288+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003561, "created_at": "2026-09-17T03:17:05.903Z" }, { "sequence": 1606, "attempt_id": "6f836cd8-bcbf-470b-9686-f42cafa71fcc", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00139275, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.039161429973319, "model_name": "gemini-3.8-flash", "output_tokens": 144, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:06.621814+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00139275, "created_at": "2026-09-17T03:17:06.622Z" }, { "sequence": 1607, "attempt_id": "a40f6dd9-c77b-44c6-855a-fa3343676be0", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00576625, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 5.269284157897346, "model_name": "muse-spark-1.3", "output_tokens": 590, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:07.233353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00576625, "created_at": "2026-09-17T03:17:07.233Z" }, { "sequence": 1608, "attempt_id": "4b99c0b5-6b15-410d-937d-4a8ea15c4060", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.6878554030554369, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:07.393522+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T03:17:07.393Z" }, { "sequence": 1609, "attempt_id": "50464778-6386-488d-91ea-fe85b8671b9d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070115, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 5.286906089051627, "model_name": "muse-spark-1.2", "output_tokens": 883, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:17:07.488966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0070115, "created_at": "2026-09-17T03:17:07.489Z" }, { "sequence": 1610, "attempt_id": "d465e382-138f-423a-a5fa-e6bc585e3d64", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0043835, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.8070053820265457, "model_name": "muse-spark-1.2", "output_tokens": 287, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:07.570693+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0043835, "created_at": "2026-09-17T03:17:07.570Z" }, { "sequence": 1611, "attempt_id": "0a0c94f3-3d55-4be1-9394-615c4115ba44", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003642, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.2458251040661708, "model_name": "gemini-3.1-pro-preview", "output_tokens": 114, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:07.746109+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003642, "created_at": "2026-09-17T03:17:07.746Z" }, { "sequence": 1612, "attempt_id": "92418b18-f466-4222-a3da-f34df69a275c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011906, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.962874019984156, "model_name": "gemini-3.1-pro-preview", "output_tokens": 790, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:17:07.888908+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011906, "created_at": "2026-09-17T03:17:07.889Z" }, { "sequence": 1613, "attempt_id": "9625da99-cdd3-4a96-83ad-303340cfcc82", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.9874364970019087, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:07.976303+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005758, "created_at": "2026-09-17T03:17:07.976Z" }, { "sequence": 1614, "attempt_id": "0c062a0f-3ee4-4156-90ce-3c606ef8866c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.171506985090673, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:08.075083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014395, "created_at": "2026-09-17T03:17:08.075Z" }, { "sequence": 1615, "attempt_id": "c94c970f-8405-4bad-866b-941c4995f539", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.2277845341013744, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:08.717110+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:17:08.717Z" }, { "sequence": 1616, "attempt_id": "769c3752-f74f-4946-b0fb-725cd6dcb316", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.6488714830484241, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:08.965927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:17:08.966Z" }, { "sequence": 1617, "attempt_id": "4a6b7e8b-a598-43be-b7e5-099c2d47b628", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.9861023450503126, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:09.051565+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003707, "created_at": "2026-09-17T03:17:09.051Z" }, { "sequence": 1618, "attempt_id": "5c0e8ec8-741b-45cf-b732-af708abe3c53", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.535188506008126, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:09.322044+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T03:17:09.322Z" }, { "sequence": 1619, "attempt_id": "68f0913f-9d9d-48e8-816b-219ca59edbf9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.8597230889135972, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:09.516387+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:17:09.516Z" }, { "sequence": 1620, "attempt_id": "6bef9bae-f365-46fa-b6bc-39d428b12c36", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005272, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.9449967839755118, "model_name": "gpt-5.6-terra", "output_tokens": 40, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:09.601164+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005272, "created_at": "2026-09-17T03:17:09.601Z" }, { "sequence": 1621, "attempt_id": "d7f4f42d-0578-489e-acf7-f1f737ed3e59", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00184425, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.7287241129670292, "model_name": "gemini-3.8-flash", "output_tokens": 263, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:09.699792+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00184425, "created_at": "2026-09-17T03:17:09.699Z" }, { "sequence": 1622, "attempt_id": "19fb8c3c-402a-4b9b-a640-914ce5d5e293", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-003", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.6951365780550987, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:10.296630+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001856, "created_at": "2026-09-17T03:17:10.296Z" }, { "sequence": 1623, "attempt_id": "ed31b28c-360d-4d7c-b7f6-edb49a02df77", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051145, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 5.008050454896875, "model_name": "muse-spark-1.3", "output_tokens": 459, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:10.389778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0051145, "created_at": "2026-09-17T03:17:10.389Z" }, { "sequence": 1624, "attempt_id": "b7eda13d-8478-4974-b782-f0f35b07667b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7819066270021722, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:11.467737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02454, "created_at": "2026-09-17T03:17:11.467Z" }, { "sequence": 1625, "attempt_id": "563f7603-c09c-49f7-b33e-4e192561d0b3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00437525, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.328463578945957, "model_name": "muse-spark-1.3", "output_tokens": 283, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:11.552367+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00437525, "created_at": "2026-09-17T03:17:11.553Z" }, { "sequence": 1626, "attempt_id": "5f3e26e0-3bc9-4248-aa60-6fa9451af1c5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01608, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.622772760922089, "model_name": "claude-opus-5", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:11.757599+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01608, "created_at": "2026-09-17T03:17:11.757Z" }, { "sequence": 1627, "attempt_id": "897fdce1-ffaa-44b9-b420-11be2fd87d6f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005416, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.5169171249726787, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:11.898696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005416, "created_at": "2026-09-17T03:17:11.898Z" }, { "sequence": 1628, "attempt_id": "5c79defb-b02d-403e-8b50-4c03688620a2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00456225, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.1803069489542395, "model_name": "muse-spark-1.2", "output_tokens": 327, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:11.987854+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00456225, "created_at": "2026-09-17T03:17:11.988Z" }, { "sequence": 1629, "attempt_id": "4df6320a-e4ae-4c3b-889a-c6adfe52d0dc", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004868, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.171445143059827, "model_name": "gemini-3.1-pro-preview", "output_tokens": 215, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:12.073785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004868, "created_at": "2026-09-17T03:17:12.075Z" }, { "sequence": 1630, "attempt_id": "46a53a81-8d83-4b20-b149-75f56185f810", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006942, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.7524967619683594, "model_name": "claude-sonnet-5", "output_tokens": 129, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:12.162229+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006942, "created_at": "2026-09-17T03:17:12.162Z" }, { "sequence": 1631, "attempt_id": "8fa6a3ea-664a-4200-8214-fcb8dc128670", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.8763752470258623, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:12.718001+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003668, "created_at": "2026-09-17T03:17:12.718Z" }, { "sequence": 1632, "attempt_id": "a38eda3e-9a74-4032-b965-aaff5a6893ef", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.9867992829531431, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:14.236147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01434, "created_at": "2026-09-17T03:17:14.236Z" }, { "sequence": 1633, "attempt_id": "fc63a95e-dfa3-4e0f-8fbc-6bec203543c2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005692000000000001, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.875215887092054, "model_name": "gpt-5.6-luna", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:14.345883+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005692000000000001, "created_at": "2026-09-17T03:17:14.346Z" }, { "sequence": 1634, "attempt_id": "792c4e8f-fcfa-4818-91d0-0562c47c562e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.7709152249153703, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:14.575447+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005736, "created_at": "2026-09-17T03:17:14.575Z" }, { "sequence": 1635, "attempt_id": "9d5af3a4-5a49-4c94-bef8-53b78751266f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00378075, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 3.2924681280273944, "model_name": "gemini-3.8-flash", "output_tokens": 782, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:14.928684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00378075, "created_at": "2026-09-17T03:17:14.928Z" }, { "sequence": 1636, "attempt_id": "d030b729-e4c3-487d-b97f-4970a0f9f889", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.7284363619983196, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:15.049767+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T03:17:15.049Z" }, { "sequence": 1637, "attempt_id": "03306168-3844-4fba-b0d8-d3d69ae2585c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.4893715570215136, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:15.651933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T03:17:15.652Z" }, { "sequence": 1638, "attempt_id": "b72a43d4-fbfc-4c3d-9e40-b9ddb1681b86", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.1758248240221292, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:15.847553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:17:15.847Z" }, { "sequence": 1639, "attempt_id": "2417717e-f480-474c-b973-cd716804c3a4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.6470684799132869, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:16.076801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:17:16.076Z" }, { "sequence": 1640, "attempt_id": "14009b0c-c0af-4d87-8c6d-c88b3b03948a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02963, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 7.111663406016305, "model_name": "claude-fable-5-1", "output_tokens": 27, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:16.163490+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02963, "created_at": "2026-09-17T03:17:16.163Z" }, { "sequence": 1641, "attempt_id": "9b91bac8-3b9d-4fbf-8d2c-3ac9e76ad073", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005266, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.4037042480194941, "model_name": "gpt-5.6-terra", "output_tokens": 40, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:16.417194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005266, "created_at": "2026-09-17T03:17:16.417Z" }, { "sequence": 1642, "attempt_id": "b0114770-84dd-4095-8894-3c81c62370e4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010916, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 7.509721216978505, "model_name": "gpt-5.6-sol", "output_tokens": 69, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:17.302793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010916, "created_at": "2026-09-17T03:17:17.302Z" }, { "sequence": 1643, "attempt_id": "7480a2dc-a31f-487a-9482-6150c6d69ba4", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.1666187830269337, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:17.388949+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:17:17.389Z" }, { "sequence": 1644, "attempt_id": "e7353464-d7e7-4ef9-b5b4-65e1a90bf2c3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.4681544459890574, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:17.602184+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:17:17.602Z" }, { "sequence": 1645, "attempt_id": "7215d3b2-fcf5-4aeb-a252-192195e81bdc", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00443525, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 1.935351073043421, "model_name": "muse-spark-1.2", "output_tokens": 298, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:18.437426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00443525, "created_at": "2026-09-17T03:17:18.437Z" }, { "sequence": 1646, "attempt_id": "225d38e6-f91e-43d0-b0a9-193722a248e4", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00141375, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 2.630488025955856, "model_name": "gemini-3.8-flash", "output_tokens": 149, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:18.561404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00141375, "created_at": "2026-09-17T03:17:18.561Z" }, { "sequence": 1647, "attempt_id": "bb8f8285-3119-4dbd-8be3-09e3d7c8fa08", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010446, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 7.216853079968132, "model_name": "gemini-3.1-pro-preview", "output_tokens": 682, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:18.769559+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010446, "created_at": "2026-09-17T03:17:18.769Z" }, { "sequence": 1648, "attempt_id": "358035d2-57b3-4565-b75f-1ee0abac6dd9", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.605119596933946, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:19.127690+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:17:19.127Z" }, { "sequence": 1649, "attempt_id": "9e56789f-5b01-4e54-9dc3-2a9d55082cfe", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.7073041759431362, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:19.393702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:17:19.393Z" }, { "sequence": 1650, "attempt_id": "fd772bb2-e7c8-4f6f-9a99-047a21eec2c2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004686, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.198214425938204, "model_name": "muse-spark-1.3", "output_tokens": 357, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:19.484486+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004686, "created_at": "2026-09-17T03:17:19.486Z" }, { "sequence": 1651, "attempt_id": "9cc45ed3-1d58-40da-9b6b-941f03a5cba6", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.182809791062027, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:19.653591+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:17:19.653Z" }, { "sequence": 1652, "attempt_id": "ba8a9362-6816-4342-b396-d83c34b0a6ff", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 2.5410386759322137, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:19.930383+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:17:19.930Z" }, { "sequence": 1653, "attempt_id": "2cc515c4-7a3f-4ed2-a03a-fb00f197c4cf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9687036889372393, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:20.182497+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:17:20.182Z" }, { "sequence": 1654, "attempt_id": "62e768b3-a988-48cf-b180-daa511563fb1", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005892, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.683150792028755, "model_name": "gemini-3.1-pro-preview", "output_tokens": 301, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:20.420067+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005892, "created_at": "2026-09-17T03:17:20.420Z" }, { "sequence": 1655, "attempt_id": "4a9a1406-60a3-4023-9f0d-abc73d0139cc", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7951852089026943, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:20.506646+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:17:20.508Z" }, { "sequence": 1656, "attempt_id": "df6928fc-a8c1-4989-ac77-2ebb4871bb2a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7912927459692582, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:20.660373+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:17:20.660Z" }, { "sequence": 1657, "attempt_id": "e3331aca-54ef-480f-8989-8bace3d30819", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.6242005520034581, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:21.640525+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:17:21.640Z" }, { "sequence": 1658, "attempt_id": "f5a753b1-ff16-4964-9598-cbb2735bae5d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001299, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.9851440729107708, "model_name": "gemini-3.8-flash", "output_tokens": 119, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:21.728672+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001299, "created_at": "2026-09-17T03:17:21.730Z" }, { "sequence": 1659, "attempt_id": "ba74d9c2-8846-409a-9c5c-4edd9786a6c0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.241458459990099, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:21.812778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:17:21.812Z" }, { "sequence": 1660, "attempt_id": "4e6b6730-53da-4d87-a96c-3f9f2faa48ee", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00327, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.6484703819733113, "model_name": "gemini-3.1-pro-preview", "output_tokens": 83, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:22.218181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00327, "created_at": "2026-09-17T03:17:22.218Z" }, { "sequence": 1661, "attempt_id": "3636b37a-c7c8-45e9-a083-3dcdac3767c1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-018", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7449686949839815, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:22.641195+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:17:22.641Z" }, { "sequence": 1662, "attempt_id": "62c8e6e8-6f68-4b2d-9b18-312c7ef72237", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00773, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 10.995992118027061, "model_name": "muse-spark-1.3", "output_tokens": 1075, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:22.984181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00773, "created_at": "2026-09-17T03:17:22.984Z" }, { "sequence": 1663, "attempt_id": "34438b61-f1ff-44c9-9be7-a9b07028a2c7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00430275, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.513616901007481, "model_name": "muse-spark-1.2", "output_tokens": 268, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:23.071824+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00430275, "created_at": "2026-09-17T03:17:23.072Z" }, { "sequence": 1664, "attempt_id": "77f414e3-6edc-4df5-8675-0af2ece1e9a7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01079425, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 12.039460727013648, "model_name": "muse-spark-1.2", "output_tokens": 1796, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:24.113577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01079425, "created_at": "2026-09-17T03:17:24.113Z" }, { "sequence": 1665, "attempt_id": "098c8e7e-4e87-4586-bbe6-c9d4e21670d0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011344, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.0623177570523694, "model_name": "gpt-5.6-sol", "output_tokens": 74, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:24.379959+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011344, "created_at": "2026-09-17T03:17:24.380Z" }, { "sequence": 1666, "attempt_id": "d6b5ee90-0040-4539-b1d9-59eacbde52a6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005808, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7589094321010634, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:24.486655+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005808, "created_at": "2026-09-17T03:17:24.486Z" }, { "sequence": 1667, "attempt_id": "0b240a8e-01c4-4970-bc07-c8a0658b3613", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02027, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 4.4942588590784, "model_name": "claude-opus-5", "output_tokens": 224, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:25.237602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02027, "created_at": "2026-09-17T03:17:25.237Z" }, { "sequence": 1668, "attempt_id": "d31f2863-30e2-4f5d-b549-92e46335b180", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-018", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8088005949975923, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:25.324371+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003789, "created_at": "2026-09-17T03:17:25.324Z" }, { "sequence": 1669, "attempt_id": "7278ef53-cff2-4aa6-8db9-7106d455643e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02706, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.5068774240789935, "model_name": "gpt-6-astra", "output_tokens": 48, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:25.407911+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02706, "created_at": "2026-09-17T03:17:25.408Z" }, { "sequence": 1670, "attempt_id": "7bb94433-dacc-42a0-92f4-17f409a111d2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0048595, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 5.154416828067042, "model_name": "muse-spark-1.3", "output_tokens": 399, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:25.493570+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0048595, "created_at": "2026-09-17T03:17:25.495Z" }, { "sequence": 1671, "attempt_id": "fde3f2b7-b75e-4bec-b3ee-956268cd2568", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03021, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.916295870905742, "model_name": "claude-fable-5-1", "output_tokens": 17, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:25.576978+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03021, "created_at": "2026-09-17T03:17:25.577Z" }, { "sequence": 1672, "attempt_id": "2fb2dcca-ef48-4015-a929-4f10a38f13ef", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005916000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.9521894060308114, "model_name": "gpt-5.6-luna", "output_tokens": 82, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:26.024326+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005916000000000001, "created_at": "2026-09-17T03:17:26.024Z" }, { "sequence": 1673, "attempt_id": "90fba422-7689-49bc-bb93-c7ec449fa392", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6663566989591345, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:26.328183+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T03:17:26.328Z" }, { "sequence": 1674, "attempt_id": "b38b1754-e28f-4d2b-8ea5-2d1272c65145", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009698, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 4.821384762064554, "model_name": "claude-sonnet-5", "output_tokens": 383, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:26.550357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009698, "created_at": "2026-09-17T03:17:26.550Z" }, { "sequence": 1675, "attempt_id": "fcbb64ba-e991-4d6c-b46f-4422741bc4ab", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00300225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.378972811042331, "model_name": "gemini-3.8-flash", "output_tokens": 558, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:26.635131+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00300225, "created_at": "2026-09-17T03:17:26.635Z" }, { "sequence": 1676, "attempt_id": "37cebe0f-42a0-4c48-9a83-3cc861d90445", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.9250608950387686, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:27.502384+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T03:17:27.502Z" }, { "sequence": 1677, "attempt_id": "8d0654c1-4cdc-48ea-9082-66c63f487728", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0386586269596592, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:27.756753+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:17:27.756Z" }, { "sequence": 1678, "attempt_id": "75457887-4c6a-4095-ad09-99d5050f0dd0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005428, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2296600160188973, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:27.865089+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005428, "created_at": "2026-09-17T03:17:27.865Z" }, { "sequence": 1679, "attempt_id": "148e8a5a-1b4a-4ae3-9653-5ef13347097d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02635, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.175622513052076, "model_name": "gpt-6-astra", "output_tokens": 49, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:28.285905+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02635, "created_at": "2026-09-17T03:17:28.286Z" }, { "sequence": 1680, "attempt_id": "3033200d-2e3d-4cb9-9fb0-09e6d9eb3e80", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.224822020973079, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:28.652528+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:17:28.652Z" }, { "sequence": 1681, "attempt_id": "6f18b9c3-f7b0-463b-b85b-244b6b988cbc", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01779, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 3.206112814019434, "model_name": "claude-opus-5", "output_tokens": 150, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:28.740439+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01779, "created_at": "2026-09-17T03:17:28.740Z" }, { "sequence": 1682, "attempt_id": "5c53d288-2496-4555-a29d-86cf56d796a2", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.352794138016179, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:28.829253+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T03:17:28.831Z" }, { "sequence": 1683, "attempt_id": "05ceca72-37ab-41ab-8f94-00107eef3dad", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.883643456036225, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:28.918848+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:17:28.919Z" }, { "sequence": 1684, "attempt_id": "fd974454-7a98-4368-a4ac-7a30f0f5f57c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012278, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 7.57854313601274, "model_name": "gemini-3.1-pro-preview", "output_tokens": 821, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:30.737433+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012278, "created_at": "2026-09-17T03:17:30.737Z" }, { "sequence": 1685, "attempt_id": "a9ee5502-5246-4bbc-965e-cd47946a1206", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0034515, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.9747401250060648, "model_name": "gemini-3.8-flash", "output_tokens": 693, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:30.826240+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0034515, "created_at": "2026-09-17T03:17:30.826Z" }, { "sequence": 1686, "attempt_id": "9eb60085-6ad4-403e-b6ba-99dc7b298796", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.780485172988847, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:30.910026+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T03:17:30.910Z" }, { "sequence": 1687, "attempt_id": "099b3302-714d-4dde-9dc3-35948eed84d3", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.2762355379527435, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:31.195430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T03:17:31.195Z" }, { "sequence": 1688, "attempt_id": "49f0fa7d-e1f8-4918-b4e6-7ade4b566386", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.6596188719850034, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:31.486151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T03:17:31.486Z" }, { "sequence": 1689, "attempt_id": "b3373ebf-8d28-4719-8a46-d87f8d54fd9b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.0373006330337375, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:32.318619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:17:32.318Z" }, { "sequence": 1690, "attempt_id": "9bbf90a3-51f6-4985-b0ff-2308f88c0f2c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007446, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.460437801899388, "model_name": "gemini-3.1-pro-preview", "output_tokens": 431, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:33.059141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007446, "created_at": "2026-09-17T03:17:33.059Z" }, { "sequence": 1691, "attempt_id": "a31ee814-e8a9-4f8e-9058-421bfe2b3de5", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.147802817984484, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:33.152927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:17:33.153Z" }, { "sequence": 1692, "attempt_id": "1a4dcbbe-34d8-4ad2-b67e-8281a61574a2", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 4.463307913974859, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:33.292871+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T03:17:33.293Z" }, { "sequence": 1693, "attempt_id": "18437fad-bfd5-42f9-89b1-8062a7a3d7ab", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.2328417559619993, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:33.805211+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:17:33.805Z" }, { "sequence": 1694, "attempt_id": "fa62fa70-4a11-4abe-ad12-307e5acb1146", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 3.0320129559841007, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:34.040381+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:17:34.040Z" }, { "sequence": 1695, "attempt_id": "c418cb79-1ac5-430e-a212-f1d3b19a6e9d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.8803062109509483, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:34.131887+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:17:34.132Z" }, { "sequence": 1696, "attempt_id": "acee6cf3-b551-49f0-abf3-99c79e3f4b56", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0013725, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3647640459239483, "model_name": "gemini-3.8-flash", "output_tokens": 143, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:34.512412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0013725, "created_at": "2026-09-17T03:17:34.512Z" }, { "sequence": 1697, "attempt_id": "b31ab253-e27e-45ee-979d-2ff87190ebee", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00730325, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 6.321386317024007, "model_name": "muse-spark-1.2", "output_tokens": 974, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:35.062170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00730325, "created_at": "2026-09-17T03:17:35.062Z" }, { "sequence": 1698, "attempt_id": "9060ba57-7333-4c8d-ad18-144aad386ab1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.7174117099493742, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:35.866300+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001884, "created_at": "2026-09-17T03:17:35.866Z" }, { "sequence": 1699, "attempt_id": "243dec30-d3a5-431a-8ae0-666893b6bee5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00459575, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.1542402839986607, "model_name": "muse-spark-1.2", "output_tokens": 344, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:36.048506+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00459575, "created_at": "2026-09-17T03:17:36.048Z" }, { "sequence": 1700, "attempt_id": "5eb019a5-8dab-4771-a0a9-2b7adfd49732", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00439, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 3.7589501620968804, "model_name": "gemini-3.1-pro-preview", "output_tokens": 180, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:36.165347+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00439, "created_at": "2026-09-17T03:17:36.165Z" }, { "sequence": 1701, "attempt_id": "3059c0c3-e063-48a9-8a2b-510097136320", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00502925, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.0977998620364815, "model_name": "muse-spark-1.3", "output_tokens": 446, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:36.475286+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00502925, "created_at": "2026-09-17T03:17:36.475Z" }, { "sequence": 1702, "attempt_id": "09096646-df2c-4b7f-ba50-eb27161a39e3", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006936, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 2.376766767934896, "model_name": "claude-sonnet-5", "output_tokens": 120, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:36.981866+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006936, "created_at": "2026-09-17T03:17:36.982Z" }, { "sequence": 1703, "attempt_id": "ddcd476d-c759-477b-a2d2-7274db782792", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00556, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.8581002879654989, "model_name": "gpt-5.6-terra", "output_tokens": 61, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:38.109700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00556, "created_at": "2026-09-17T03:17:38.109Z" }, { "sequence": 1704, "attempt_id": "e53f5563-8090-43d5-b098-5c37fe0c07a0", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02935, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 4.189949057996273, "model_name": "claude-fable-5-1", "output_tokens": 13, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:38.322166+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02935, "created_at": "2026-09-17T03:17:38.322Z" }, { "sequence": 1705, "attempt_id": "79fb01ae-9816-4a67-85d4-6f7ace925bb3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005644, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.468808818026446, "model_name": "gpt-5.6-luna", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:39.032347+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005644, "created_at": "2026-09-17T03:17:39.032Z" }, { "sequence": 1706, "attempt_id": "3c89e424-db17-46fc-abb7-f88c1f3871a4", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.020665, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 5.190015550004318, "model_name": "claude-opus-5", "output_tokens": 253, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:39.411727+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.020665, "created_at": "2026-09-17T03:17:39.411Z" }, { "sequence": 1707, "attempt_id": "c28c5df4-7840-44c9-8189-b9e57b04a178", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 1.5356252730125561, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:17:39.945206+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003761, "created_at": "2026-09-17T03:17:39.945Z" }, { "sequence": 1708, "attempt_id": "0c6b60ce-e4fd-453c-8fea-bbc0a8939264", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00247275, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 1.9101667359936982, "model_name": "gemini-3.8-flash", "output_tokens": 427, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:17:40.108630+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00247275, "created_at": "2026-09-17T03:17:40.108Z" }, { "sequence": 1709, "attempt_id": "dee3ba87-9712-42a4-b9ea-7994c5b8108c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02629, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 4.243764049955644, "model_name": "gpt-6-astra", "output_tokens": 43, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:40.201712+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02629, "created_at": "2026-09-17T03:17:40.201Z" }, { "sequence": 1710, "attempt_id": "da87462d-5402-4a04-92b7-a1d46fbcdf8c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.013496, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 4.505167815950699, "model_name": "gpt-5.6-sol", "output_tokens": 192, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:40.652205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.013496, "created_at": "2026-09-17T03:17:40.652Z" }, { "sequence": 1711, "attempt_id": "e4953d71-f46c-419d-97ab-140a53cae304", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-014", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.6779803119134158, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:17:41.422161+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:17:41.422Z" }, { "sequence": 1712, "attempt_id": "cab827fe-ff32-4cf9-b535-eb913399177c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 2.552975346916355, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:42.591417+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:17:42.591Z" }, { "sequence": 1713, "attempt_id": "f28a7217-5eb8-46c6-9d3e-fb8dfb83e468", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.8258687130874023, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:43.343058+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:17:43.343Z" }, { "sequence": 1714, "attempt_id": "91cbd21e-ec1c-48a0-aa52-9ee8f25fa810", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005796, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7413520780391991, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:45.174037+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005796, "created_at": "2026-09-17T03:17:45.174Z" }, { "sequence": 1715, "attempt_id": "8f62259a-2b36-41ce-a735-757eaf401df1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011564, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.23754173098132, "model_name": "gpt-5.6-sol", "output_tokens": 85, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:45.930222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011564, "created_at": "2026-09-17T03:17:45.930Z" }, { "sequence": 1716, "attempt_id": "546a56b4-618f-457d-8642-40181d4e62d5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02292, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.889642588910647, "model_name": "claude-opus-5", "output_tokens": 330, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:46.091696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02292, "created_at": "2026-09-17T03:17:46.091Z" }, { "sequence": 1717, "attempt_id": "5505c829-734d-40db-89fa-e51dea7a988f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009339, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 17.99480169499293, "model_name": "muse-spark-1.3", "output_tokens": 1453, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:46.369676+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009339, "created_at": "2026-09-17T03:17:46.369Z" }, { "sequence": 1718, "attempt_id": "00501d94-256c-4d96-9026-0e2b061ccd98", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-014", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.9344471249496564, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:17:47.393644+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:17:47.393Z" }, { "sequence": 1719, "attempt_id": "36a66667-27ee-48b5-97fe-1a22b64e4167", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006504000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.459346337011084, "model_name": "gpt-5.6-luna", "output_tokens": 131, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:47.723449+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006504000000000001, "created_at": "2026-09-17T03:17:47.723Z" }, { "sequence": 1720, "attempt_id": "11cc57b7-23ff-4eb3-a61d-20defdce621e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.019753, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 22.514183121034876, "model_name": "muse-spark-1.2", "output_tokens": 3881, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:47.838835+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.019753, "created_at": "2026-09-17T03:17:47.838Z" }, { "sequence": 1721, "attempt_id": "7b957fc7-e698-4175-a119-319722ee157f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00195975, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.809975539916195, "model_name": "gemini-3.8-flash", "output_tokens": 280, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:17:47.993125+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00195975, "created_at": "2026-09-17T03:17:47.993Z" }, { "sequence": 1722, "attempt_id": "24a33e84-b3a4-4f3f-a65e-ea79f8671515", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.4140214830404148, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:50.340118+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T03:17:50.340Z" }, { "sequence": 1723, "attempt_id": "64d134fe-c76e-4c67-a354-3f8bcd66f494", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015595, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.767409566906281, "model_name": "claude-opus-5", "output_tokens": 59, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:50.849037+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015595, "created_at": "2026-09-17T03:17:50.849Z" }, { "sequence": 1724, "attempt_id": "efc0c185-a622-4fdc-aa3d-16890b7d682e", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.6704592190217227, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:51.609459+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T03:17:51.609Z" }, { "sequence": 1725, "attempt_id": "3e2338ec-b9d3-4760-ad43-4be65659323a", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008438, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.706884339102544, "model_name": "gemini-3.1-pro-preview", "output_tokens": 501, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:51.732188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008438, "created_at": "2026-09-17T03:17:51.732Z" }, { "sequence": 1726, "attempt_id": "5cd9ecea-d5b5-4d34-a018-853b304428d3", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015628, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 11.443716884008609, "model_name": "claude-sonnet-5", "output_tokens": 976, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:51.834686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015628, "created_at": "2026-09-17T03:17:51.834Z" }, { "sequence": 1727, "attempt_id": "22da910d-ed01-45c3-a5f9-34b1ed2e299c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.806738344952464, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:52.236220+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005758, "created_at": "2026-09-17T03:17:52.236Z" }, { "sequence": 1728, "attempt_id": "e51f9828-eca3-4b90-a79b-f94478ea91a1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.0888699559727684, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:52.923935+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:17:52.924Z" }, { "sequence": 1729, "attempt_id": "6ad6d576-f1d9-4fb7-951d-37dca37eaf21", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005368, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.3431439000414684, "model_name": "gpt-5.6-terra", "output_tokens": 48, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:53.271032+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005368, "created_at": "2026-09-17T03:17:53.271Z" }, { "sequence": 1730, "attempt_id": "4782dcda-a926-4858-b0fd-b4d53c0ab1c4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.9490162800066173, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:53.364740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:17:53.366Z" }, { "sequence": 1731, "attempt_id": "c1558742-eb99-4680-91d6-2960184b951a", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.8469828380038962, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:54.312272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003707, "created_at": "2026-09-17T03:17:54.312Z" }, { "sequence": 1732, "attempt_id": "0987ae45-7e3b-4d89-ba3d-cacd68f318f9", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.028616, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 17.442417040932924, "model_name": "gemini-3.1-pro-preview", "output_tokens": 2191, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:54.512515+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.028616, "created_at": "2026-09-17T03:17:54.512Z" }, { "sequence": 1733, "attempt_id": "2a42914b-ba2d-437d-a2f1-f5324003d796", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0017055, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.4621168039739132, "model_name": "gemini-3.8-flash", "output_tokens": 226, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:54.827185+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0017055, "created_at": "2026-09-17T03:17:54.827Z" }, { "sequence": 1734, "attempt_id": "885a2174-b0c0-4935-8157-8b80ebef7494", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 3.360468010068871, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:55.060413+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:17:55.060Z" }, { "sequence": 1735, "attempt_id": "9257af6f-c82f-41b4-9242-a98d71925c62", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00709225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 7.600743173970841, "model_name": "muse-spark-1.3", "output_tokens": 902, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:55.152861+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00709225, "created_at": "2026-09-17T03:17:55.153Z" }, { "sequence": 1736, "attempt_id": "fc6a07ea-2d0a-4cb2-90e2-f155fa43f807", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003932, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 2.7748812650097534, "model_name": "gemini-3.1-pro-preview", "output_tokens": 137, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:55.794395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003932, "created_at": "2026-09-17T03:17:55.794Z" }, { "sequence": 1737, "attempt_id": "2d038b04-1f4a-4fab-802d-a8d7d7f18cef", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01543075, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 31.501432609045878, "model_name": "muse-spark-1.3", "output_tokens": 2864, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:56.072961+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01543075, "created_at": "2026-09-17T03:17:56.073Z" }, { "sequence": 1738, "attempt_id": "e198f3d8-c3fa-4ab5-98ea-f4bae1a0e018", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6939985209610313, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:56.578230+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:17:56.578Z" }, { "sequence": 1739, "attempt_id": "76a2b944-a1c8-44ce-9473-764cb9749035", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0046685, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.121943050995469, "model_name": "muse-spark-1.2", "output_tokens": 352, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:56.724458+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0046685, "created_at": "2026-09-17T03:17:56.724Z" }, { "sequence": 1740, "attempt_id": "ed8284b9-615f-45c6-bdb7-2a1b083c674c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.268042938085273, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:57.948025+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:17:57.948Z" }, { "sequence": 1741, "attempt_id": "d89f27fa-705c-41be-8ddd-4510f48796cd", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015845, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.8246711120009422, "model_name": "claude-opus-5", "output_tokens": 66, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:58.043686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015845, "created_at": "2026-09-17T03:17:58.043Z" }, { "sequence": 1742, "attempt_id": "2250c7d1-968b-4375-b478-4e72fc2f08e8", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.8374732219381258, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:58.130606+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:17:58.130Z" }, { "sequence": 1743, "attempt_id": "5832d5a2-1cd0-44bf-9425-56acf72dd8c5", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007058, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.854242269997485, "model_name": "claude-sonnet-5", "output_tokens": 138, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:58.219523+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007058, "created_at": "2026-09-17T03:17:58.220Z" }, { "sequence": 1744, "attempt_id": "38408e3f-a34e-42c8-9e61-610dc0de4173", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.49603548203595, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:58.412455+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:17:58.412Z" }, { "sequence": 1745, "attempt_id": "7d5ec978-ba82-49a3-8d88-3a7760cb1756", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005616, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9577694750623778, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:58.768677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005616, "created_at": "2026-09-17T03:17:58.768Z" }, { "sequence": 1746, "attempt_id": "ec38d6b0-654d-4285-831e-b3f763a04924", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0046685, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 4.698496990953572, "model_name": "muse-spark-1.3", "output_tokens": 352, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:59.099425+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0046685, "created_at": "2026-09-17T03:17:59.099Z" }, { "sequence": 1747, "attempt_id": "61145a7a-d3ec-457e-8ddf-d3806026cacf", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.0026776890736073, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:59.312257+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:17:59.312Z" }, { "sequence": 1748, "attempt_id": "56104778-e8b6-4ea9-88af-caefb30272a6", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00231375, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.23195147595834, "model_name": "gemini-3.8-flash", "output_tokens": 387, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:00.451801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00231375, "created_at": "2026-09-17T03:18:00.451Z" }, { "sequence": 1749, "attempt_id": "e1266bdd-1cc5-4201-95c5-2514ab16bc18", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00057, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.497011675965041, "model_name": "gpt-5.6-luna", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:00.542392+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00057, "created_at": "2026-09-17T03:18:00.542Z" }, { "sequence": 1750, "attempt_id": "e3aef8df-d76a-4964-aa85-323c158b70c2", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.5200565040577203, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:01.150904+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T03:18:01.151Z" }, { "sequence": 1751, "attempt_id": "7514fbf6-d259-464f-a627-e68a0c9a50ce", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005024, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.557848635944538, "model_name": "gemini-3.1-pro-preview", "output_tokens": 227, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:01.775197+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005024, "created_at": "2026-09-17T03:18:01.775Z" }, { "sequence": 1752, "attempt_id": "21384e33-1e5a-49b4-9439-7a02cb4ded35", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.4617198299383745, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:01.867415+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T03:18:01.867Z" }, { "sequence": 1753, "attempt_id": "5ea3cba6-5456-411c-af7c-a28d126101a8", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.6666506249457598, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:02.209408+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:18:02.209Z" }, { "sequence": 1754, "attempt_id": "c45018bb-ab77-4636-b3eb-81fe456d677a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.3935657660476863, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:02.582880+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T03:18:02.583Z" }, { "sequence": 1755, "attempt_id": "c3507ff1-f20b-410b-8163-a03c88ea125d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0773984170518816, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:03.036996+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:18:03.037Z" }, { "sequence": 1756, "attempt_id": "ba994c62-1640-4cdd-bccd-3554aa9bbede", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.8260056159924716, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:03.131301+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:18:03.131Z" }, { "sequence": 1757, "attempt_id": "46ad9117-0064-4dea-a828-84f2a65faf95", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.799745506956242, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:03.220667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:18:03.220Z" }, { "sequence": 1758, "attempt_id": "df800743-a7c2-4f28-9109-d5307db0b5de", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0058495, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.30409041501116, "model_name": "muse-spark-1.2", "output_tokens": 629, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:03.313124+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0058495, "created_at": "2026-09-17T03:18:03.313Z" }, { "sequence": 1759, "attempt_id": "df0ebc20-0171-4c32-8e9a-a93d656dd5c8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.3513225850183517, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:03.403418+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:18:03.403Z" }, { "sequence": 1760, "attempt_id": "6a577773-511b-4367-91da-6466cf18908d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01464025, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 15.955028312979266, "model_name": "muse-spark-1.2", "output_tokens": 2678, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:03.767819+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01464025, "created_at": "2026-09-17T03:18:03.768Z" }, { "sequence": 1761, "attempt_id": "29e9d558-75f9-4897-9340-514be3b014fb", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.794377721962519, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:18:04.105499+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:18:04.105Z" }, { "sequence": 1762, "attempt_id": "5442af2a-7e2b-4e11-a9db-3abbbac2a2fe", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.244947528000921, "model_name": "gemini-3.8-flash", "output_tokens": 113, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:04.376572+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00125925, "created_at": "2026-09-17T03:18:04.376Z" }, { "sequence": 1763, "attempt_id": "807e4370-83fa-40d3-84d6-eba23fd257a6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.689378670998849, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:05.170905+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001856, "created_at": "2026-09-17T03:18:05.171Z" }, { "sequence": 1764, "attempt_id": "b7fa8773-5ad0-46e4-a9fe-426ba5e775b2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004291, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 2.7506123109487817, "model_name": "muse-spark-1.3", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:06.064053+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004291, "created_at": "2026-09-17T03:18:06.064Z" }, { "sequence": 1765, "attempt_id": "7abe6d1f-c4a4-4b43-a8df-03f453e70c36", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006532, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.1971416290616617, "model_name": "claude-sonnet-5", "output_tokens": 88, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:06.393969+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006532, "created_at": "2026-09-17T03:18:06.394Z" }, { "sequence": 1766, "attempt_id": "574b0b95-f9c1-4676-b407-9c9b3bb1c42e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.66246646491345, "model_name": "claude-opus-5", "output_tokens": 102, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:07.522619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01668, "created_at": "2026-09-17T03:18:07.522Z" }, { "sequence": 1767, "attempt_id": "a37ffeae-8cbf-4775-a274-656ecad6d7cb", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007328, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 5.328509305021726, "model_name": "gemini-3.1-pro-preview", "output_tokens": 425, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:08.004235+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007328, "created_at": "2026-09-17T03:18:08.004Z" }, { "sequence": 1768, "attempt_id": "5521fa8e-94f9-4b14-9ae9-0fecce096213", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.866269055986777, "model_name": "gpt-5.6-sol", "output_tokens": 66, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:08.098757+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010856, "created_at": "2026-09-17T03:18:08.099Z" }, { "sequence": 1769, "attempt_id": "c406aca9-6e43-46ca-9b20-67801a19f83d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02679, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.94752228888683, "model_name": "gpt-6-astra", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:08.220508+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02679, "created_at": "2026-09-17T03:18:08.220Z" }, { "sequence": 1770, "attempt_id": "b0c631b9-c9fc-4a14-9188-b1c43bf03150", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005728, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.065900538000278, "model_name": "gpt-5.6-terra", "output_tokens": 80, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:08.550406+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005728, "created_at": "2026-09-17T03:18:08.550Z" }, { "sequence": 1771, "attempt_id": "b4c51741-d800-44b8-8397-96caed8b24fc", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02963, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.374888858990744, "model_name": "claude-fable-5-1", "output_tokens": 27, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:08.871596+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02963, "created_at": "2026-09-17T03:18:08.871Z" }, { "sequence": 1772, "attempt_id": "ec967940-f380-4a09-8522-ee5aee8e125d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 1.0106478090165183, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:09.320305+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003668, "created_at": "2026-09-17T03:18:09.320Z" }, { "sequence": 1773, "attempt_id": "33b47cec-074f-4673-b040-b7431df46087", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005524, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7926646081032231, "model_name": "gpt-5.6-luna", "output_tokens": 63, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:09.411441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005524, "created_at": "2026-09-17T03:18:09.411Z" }, { "sequence": 1774, "attempt_id": "2c6496a7-bf26-43f9-baeb-2138d3b41347", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.003177, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 2.6632636020658538, "model_name": "gemini-3.8-flash", "output_tokens": 621, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:18:10.855018+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003177, "created_at": "2026-09-17T03:18:10.855Z" }, { "sequence": 1775, "attempt_id": "a163f291-78b7-4da7-aa53-5946677bb7c7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00398925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 7.706908491090871, "model_name": "muse-spark-1.2", "output_tokens": 201, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:11.201441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00398925, "created_at": "2026-09-17T03:18:11.201Z" }, { "sequence": 1776, "attempt_id": "ddada721-a616-4866-a418-b82b7c37ebcc", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9645242699189112, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:11.466622+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T03:18:11.466Z" }, { "sequence": 1777, "attempt_id": "20ea34eb-42f1-4839-880d-4423a34bb71f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0082635, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 14.070832891971804, "model_name": "muse-spark-1.3", "output_tokens": 1197, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:12.576009+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0082635, "created_at": "2026-09-17T03:18:12.576Z" }, { "sequence": 1778, "attempt_id": "f6b66a20-aed5-4318-8465-2b079a2c3284", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 1.6219840409466997, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:12.913944+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:18:12.914Z" }, { "sequence": 1779, "attempt_id": "91ca7e97-63ce-4cd9-83c7-0c04c67665d4", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00612, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.406928612035699, "model_name": "claude-sonnet-5", "output_tokens": 42, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:13.363232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00612, "created_at": "2026-09-17T03:18:13.363Z" }, { "sequence": 1780, "attempt_id": "a2f9a90f-9290-4c2f-90ff-809f2c22f07e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.9119623580481857, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:13.467648+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:18:13.467Z" }, { "sequence": 1781, "attempt_id": "9badc219-0a4e-4b4d-8fbe-825bcac7b219", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00528, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.3667318450752646, "model_name": "gpt-5.6-terra", "output_tokens": 37, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:14.368153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00528, "created_at": "2026-09-17T03:18:14.368Z" }, { "sequence": 1782, "attempt_id": "44790442-6b3a-4786-b48b-cea858f2b901", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.002324577071704, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:14.458895+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:18:14.459Z" }, { "sequence": 1783, "attempt_id": "7c9ab018-1ec3-4a3e-bb8b-a809850a265d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01392625, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 35.349566776072606, "model_name": "muse-spark-1.3", "output_tokens": 2525, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:14.546816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01392625, "created_at": "2026-09-17T03:18:14.547Z" }, { "sequence": 1784, "attempt_id": "ed0263d4-9ed0-4e90-8048-e9b70260c135", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.6568453561048955, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:15.322369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:18:15.322Z" }, { "sequence": 1785, "attempt_id": "a90bad8a-830f-4d60-b08c-63a44e5b6940", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.9758359850384295, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:18:15.522911+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003764, "created_at": "2026-09-17T03:18:15.523Z" }, { "sequence": 1786, "attempt_id": "c3957545-0041-4ad0-a39b-00c256a4c1e0", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01449, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 8.885141520993784, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1019, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:16.984249+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01449, "created_at": "2026-09-17T03:18:16.984Z" }, { "sequence": 1787, "attempt_id": "ea64e585-f126-4c52-9377-6a230ae0f46d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003591, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.357607073965482, "model_name": "gemini-3.8-flash", "output_tokens": 725, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:18.816821+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003591, "created_at": "2026-09-17T03:18:18.817Z" }, { "sequence": 1788, "attempt_id": "d2e2683f-c82b-423d-a2b1-1637a76f0451", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01657, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.1098716150736436, "model_name": "claude-opus-5", "output_tokens": 95, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:20.185285+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01657, "created_at": "2026-09-17T03:18:20.185Z" }, { "sequence": 1789, "attempt_id": "e187d5b5-7613-43c3-b682-3e26028a32b4", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 11.16955458605662, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:20.581292+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:18:20.581Z" }, { "sequence": 1790, "attempt_id": "54a2821b-9c1c-4acd-9d36-07df17dcd9ba", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.5552927440730855, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:20.832937+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:18:20.833Z" }, { "sequence": 1791, "attempt_id": "26b87f4f-1a94-49fe-9b6e-ca32f670ba45", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010138, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 7.3135864429641515, "model_name": "gemini-3.1-pro-preview", "output_tokens": 651, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:20.922069+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010138, "created_at": "2026-09-17T03:18:20.923Z" }, { "sequence": 1792, "attempt_id": "eea36abf-d9bf-42d6-96d9-ef4dfdb6dd13", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006598, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.3516844959231094, "model_name": "claude-sonnet-5", "output_tokens": 92, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:21.259662+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006598, "created_at": "2026-09-17T03:18:21.259Z" }, { "sequence": 1793, "attempt_id": "be4ed3e9-c8e3-477c-b641-a6234966cd91", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.034696, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 42.38084592693485, "model_name": "muse-spark-1.2", "output_tokens": 7412, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:21.883411+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.034696, "created_at": "2026-09-17T03:18:21.883Z" }, { "sequence": 1794, "attempt_id": "a5c9ff02-d725-49ba-aa97-d5aec76da670", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.3696132869226858, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:22.291955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:18:22.292Z" }, { "sequence": 1795, "attempt_id": "fdf43fd2-00c3-4abd-8d0c-753b387584a4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6572157909395173, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:22.382996+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:18:22.383Z" }, { "sequence": 1796, "attempt_id": "1122b7ac-25cb-4ddf-a5b7-03fb0797ee98", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00546, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.458061649929732, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:22.470555+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00546, "created_at": "2026-09-17T03:18:22.470Z" }, { "sequence": 1797, "attempt_id": "98cd321a-7226-45e7-8a2f-92a60b61dd55", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.8219655540306121, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:23.292838+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:18:23.293Z" }, { "sequence": 1798, "attempt_id": "a706f17e-b0be-419b-8172-bec275dd9761", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005556000000000001, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.4640162619762123, "model_name": "gpt-5.6-luna", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:24.812135+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005556000000000001, "created_at": "2026-09-17T03:18:24.812Z" }, { "sequence": 1799, "attempt_id": "8ed8e7ed-e797-4e38-8c95-485b8eb83746", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 9.546847705962136, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:25.167516+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:18:25.167Z" }, { "sequence": 1800, "attempt_id": "94235a84-f615-450c-a202-a493bd20a85c", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00214875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.7132873149821535, "model_name": "gemini-3.8-flash", "output_tokens": 343, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:26.096626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00214875, "created_at": "2026-09-17T03:18:26.096Z" }, { "sequence": 1801, "attempt_id": "63a2ea9b-ccb8-45b2-bbb9-4ce4f782f6fe", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006692, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.863794052042067, "model_name": "gemini-3.1-pro-preview", "output_tokens": 366, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:26.846173+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006692, "created_at": "2026-09-17T03:18:26.846Z" }, { "sequence": 1802, "attempt_id": "9ce71385-1dd7-47c9-903c-2d584747562f", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.783692136988975, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:27.042374+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01421, "created_at": "2026-09-17T03:18:27.042Z" }, { "sequence": 1803, "attempt_id": "8bd1fe07-88a3-47a3-8f69-d2ba20179403", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.6068368430715054, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:27.546243+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001833, "created_at": "2026-09-17T03:18:27.546Z" }, { "sequence": 1804, "attempt_id": "77e0755b-1346-420f-bc18-bb00017be293", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 2.7706054889131337, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:27.674730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02844, "created_at": "2026-09-17T03:18:27.674Z" }, { "sequence": 1805, "attempt_id": "5d2d976d-5a3e-4ef9-bd3f-41c61210e622", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.9150350649142638, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:28.103725+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005684, "created_at": "2026-09-17T03:18:28.103Z" }, { "sequence": 1806, "attempt_id": "1237b5f2-9478-4ee9-92ae-de29bcbd1fa5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00650825, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.82626821892336, "model_name": "muse-spark-1.2", "output_tokens": 784, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:28.221668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00650825, "created_at": "2026-09-17T03:18:28.221Z" }, { "sequence": 1807, "attempt_id": "1c9ec14b-ca70-4f52-8fac-ece4566f7dd3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.9930016229627654, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:28.753141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004904, "created_at": "2026-09-17T03:18:28.753Z" }, { "sequence": 1808, "attempt_id": "25fba717-d893-49d3-a2e3-93d1fcc7b7ba", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.6621509689139202, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:28.845216+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02438, "created_at": "2026-09-17T03:18:28.846Z" }, { "sequence": 1809, "attempt_id": "1ca52d72-df41-40f2-99fe-ce36647a367a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005399, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.209897020016797, "model_name": "muse-spark-1.3", "output_tokens": 523, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:28.930938+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005399, "created_at": "2026-09-17T03:18:28.931Z" }, { "sequence": 1810, "attempt_id": "662dc15d-b4e2-431d-a510-8732f0a79478", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.9374549520434812, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:29.130763+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004904, "created_at": "2026-09-17T03:18:29.130Z" }, { "sequence": 1811, "attempt_id": "8ad1444b-8ae6-4d17-a437-a5a71897d205", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.8377444299403578, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:29.474436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009752, "created_at": "2026-09-17T03:18:29.474Z" }, { "sequence": 1812, "attempt_id": "640af946-af9e-48dc-9ecd-7def52653e3a", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003495, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.8828969240421429, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:29.911796+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003495, "created_at": "2026-09-17T03:18:29.911Z" }, { "sequence": 1813, "attempt_id": "b7982d06-155d-471f-98d6-e30351140ba1", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0011775, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3620123430155218, "model_name": "gemini-3.8-flash", "output_tokens": 91, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:30.207534+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0011775, "created_at": "2026-09-17T03:18:30.207Z" }, { "sequence": 1814, "attempt_id": "0e52fed9-668b-4f59-8a3e-1ae788daeaa8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00379075, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.6444625149015337, "model_name": "muse-spark-1.2", "output_tokens": 154, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:30.863437+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00379075, "created_at": "2026-09-17T03:18:30.863Z" }, { "sequence": 1815, "attempt_id": "ac1ffccc-f1a7-40c2-83b0-7cc04e7257a0", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01553225, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 15.741402006009594, "model_name": "muse-spark-1.2", "output_tokens": 2902, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:31.154959+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01553225, "created_at": "2026-09-17T03:18:31.155Z" }, { "sequence": 1816, "attempt_id": "f9e3ebc9-0d80-4bf5-8ed0-5137058d6fd9", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6877396159106866, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:31.641835+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:18:31.641Z" }, { "sequence": 1817, "attempt_id": "8c39acdb-c8db-40c7-a321-2f7c25722aba", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01208625, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 23.29569342499599, "model_name": "muse-spark-1.3", "output_tokens": 2100, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:31.937511+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01208625, "created_at": "2026-09-17T03:18:31.937Z" }, { "sequence": 1818, "attempt_id": "d7273d04-11bc-4c37-839d-b45a60361256", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6547739380039275, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:32.031426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:18:32.031Z" }, { "sequence": 1819, "attempt_id": "086ff004-ba40-4f5a-8a61-68f302d534e5", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005842, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.079863735008985, "model_name": "gemini-3.1-pro-preview", "output_tokens": 301, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:32.387438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005842, "created_at": "2026-09-17T03:18:32.387Z" }, { "sequence": 1820, "attempt_id": "53575b8b-53d6-4b5f-afa7-38695cc5a445", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00456425, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 3.4436037900159135, "model_name": "muse-spark-1.3", "output_tokens": 336, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:32.477879+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00456425, "created_at": "2026-09-17T03:18:32.478Z" }, { "sequence": 1821, "attempt_id": "cd353427-8135-46ed-8e17-853b70952c9a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0093668809859082, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:32.741824+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:18:32.741Z" }, { "sequence": 1822, "attempt_id": "393f34d9-cfee-4cd2-87fe-570bbd700bb6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012757, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 18.30742082395591, "model_name": "muse-spark-1.3", "output_tokens": 2249, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:32.942267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012757, "created_at": "2026-09-17T03:18:32.942Z" }, { "sequence": 1823, "attempt_id": "6e662405-222d-40e8-be12-1754bc14f026", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.99567639909219, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:33.030671+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:18:33.030Z" }, { "sequence": 1824, "attempt_id": "6e062996-b8da-4f05-8637-b050a5d4201e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.164008477004245, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:33.406847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:18:33.406Z" }, { "sequence": 1825, "attempt_id": "56e8a471-e982-4a5a-bb74-9e26cc83bb70", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.020814697956666, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:33.582774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:18:33.582Z" }, { "sequence": 1826, "attempt_id": "a0376dce-1c30-4dd0-af3e-404f7bec97f0", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8289966660086066, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:33.671657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:18:33.671Z" }, { "sequence": 1827, "attempt_id": "72c57e5b-4115-4943-86f1-08e9cb94dbfd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.8938366930233315, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:34.013116+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:18:34.013Z" }, { "sequence": 1828, "attempt_id": "3ca4b0f6-2c92-466b-b91f-43b41ccec598", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00134025, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.7493590359808877, "model_name": "gemini-3.8-flash", "output_tokens": 130, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:34.314504+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00134025, "created_at": "2026-09-17T03:18:34.314Z" }, { "sequence": 1829, "attempt_id": "6efaa5e6-dc93-4f76-beda-b2aabc6ebaaa", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.0793831530027092, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:35.182436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:18:35.182Z" }, { "sequence": 1830, "attempt_id": "d5523abb-870b-401a-88e2-47746424e715", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0042985, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.179440753068775, "model_name": "muse-spark-1.3", "output_tokens": 267, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:35.275439+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0042985, "created_at": "2026-09-17T03:18:35.275Z" }, { "sequence": 1831, "attempt_id": "bd8cf360-0bb2-4480-bbf1-242e4ada15e8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00446, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.3023848199518397, "model_name": "muse-spark-1.2", "output_tokens": 305, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:35.421200+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00446, "created_at": "2026-09-17T03:18:35.421Z" }, { "sequence": 1832, "attempt_id": "5f0d3789-7ddd-488c-916b-8e65afb1c645", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.0284659470198676, "model_name": "gemini-3.1-pro-preview", "output_tokens": 124, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:35.509220+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003762, "created_at": "2026-09-17T03:18:35.510Z" }, { "sequence": 1833, "attempt_id": "76fc0153-93e3-4cb3-a380-b6ec90b3d30d", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9379101989325136, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:35.698082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:18:35.698Z" }, { "sequence": 1834, "attempt_id": "ede08040-7ece-4e55-89ed-6c0881def201", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.056459540966898, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:35.788161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:18:35.788Z" }, { "sequence": 1835, "attempt_id": "c1c1da9b-5238-41c0-a417-be9f6b9fd767", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7003113819519058, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:36.104520+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:18:36.104Z" }, { "sequence": 1836, "attempt_id": "c498d4c3-394f-4278-a4c2-9c25f2b20ff5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0697254149708897, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:36.345441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:18:36.345Z" }, { "sequence": 1837, "attempt_id": "79b6b09e-cdbd-4502-84f5-2b36f7ca71c7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9976366650080308, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:36.507130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:18:36.507Z" }, { "sequence": 1838, "attempt_id": "51e5a6c9-9bfd-469b-90b4-ee9e4bbd6ad9", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.2013862510211766, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:36.697723+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:18:36.697Z" }, { "sequence": 1839, "attempt_id": "2cebf64f-ab25-4565-aed0-db401ac4406e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.3819021939998493, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:36.787831+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:18:36.788Z" }, { "sequence": 1840, "attempt_id": "1ff61193-964a-4342-ab58-72d74932f38d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02181025, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 27.824937036028132, "model_name": "muse-spark-1.2", "output_tokens": 4388, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:36.880286+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02181025, "created_at": "2026-09-17T03:18:36.880Z" }, { "sequence": 1841, "attempt_id": "e9830563-91b6-4252-b76b-5dcec78ce022", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00113925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.0215410640230402, "model_name": "gemini-3.8-flash", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:36.969147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00113925, "created_at": "2026-09-17T03:18:36.969Z" }, { "sequence": 1842, "attempt_id": "e0c6e9be-8619-4d93-8be4-e1d33f077367", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.0899517909856513, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:37.059689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:18:37.059Z" }, { "sequence": 1843, "attempt_id": "e601994f-2ac4-4b23-a6b4-aa2b925c93f7", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001857, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.7300039129331708, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:37.786779+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001857, "created_at": "2026-09-17T03:18:37.786Z" }, { "sequence": 1844, "attempt_id": "79477f97-6ab3-42c4-a057-01eb3a32a733", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003384, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.3626575289526954, "model_name": "gemini-3.1-pro-preview", "output_tokens": 93, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:37.972602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003384, "created_at": "2026-09-17T03:18:37.972Z" }, { "sequence": 1845, "attempt_id": "122d2f91-e2bd-4600-ad1e-4f0f8a851a48", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004163, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6414696569554508, "model_name": "muse-spark-1.2", "output_tokens": 236, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:38.076205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004163, "created_at": "2026-09-17T03:18:38.076Z" }, { "sequence": 1846, "attempt_id": "a660279f-28f8-4564-b8b9-b1ced01b7eb0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.1663204570068046, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:38.316394+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:18:38.316Z" }, { "sequence": 1847, "attempt_id": "5e46510c-6655-4c72-afff-a489aa3d954d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.565968323033303, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:38.625919+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:18:38.626Z" }, { "sequence": 1848, "attempt_id": "5801a5fb-5397-409e-b2b3-5487c90b162d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 8.837906496948563, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:38.840364+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:18:38.840Z" }, { "sequence": 1849, "attempt_id": "2f8ff4ef-f3a7-4ff4-8ecf-e076224aaf8a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.8545646149432287, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:38.930628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:18:38.930Z" }, { "sequence": 1850, "attempt_id": "598d8710-5ac0-4dbc-9f5d-39ff082e7ad9", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006498, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.2913989609805867, "model_name": "claude-sonnet-5", "output_tokens": 85, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:39.171953+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006498, "created_at": "2026-09-17T03:18:39.172Z" }, { "sequence": 1851, "attempt_id": "d5479fb2-e5fe-47b4-aabf-50707d3443b3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00471975, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.389003223972395, "model_name": "muse-spark-1.3", "output_tokens": 367, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:39.586063+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00471975, "created_at": "2026-09-17T03:18:39.586Z" }, { "sequence": 1852, "attempt_id": "461f8f57-80fe-46f0-a072-9c43a1485019", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.9523923260858282, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:39.678673+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003707, "created_at": "2026-09-17T03:18:39.679Z" }, { "sequence": 1853, "attempt_id": "f4b58443-a76d-41c6-9e56-9307a881281b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005404, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.11553031206131, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:39.998492+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005404, "created_at": "2026-09-17T03:18:39.998Z" }, { "sequence": 1854, "attempt_id": "b0d493a5-b003-4a10-91e7-f74cadbef85b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01767, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 3.382683486910537, "model_name": "claude-opus-5", "output_tokens": 142, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:40.170784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01767, "created_at": "2026-09-17T03:18:40.170Z" }, { "sequence": 1855, "attempt_id": "f8c7b060-f17c-40fc-b4a0-9deada5304cd", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00182925, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 2.015661451034248, "model_name": "gemini-3.8-flash", "output_tokens": 259, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:40.424395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00182925, "created_at": "2026-09-17T03:18:40.424Z" }, { "sequence": 1856, "attempt_id": "855c2dfd-b287-4908-bb3f-b939a613b6bf", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.7020874910522252, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:40.794304+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T03:18:40.794Z" }, { "sequence": 1857, "attempt_id": "4855c266-f170-49da-807f-40d06d24bbc0", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 4.36707341496367, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:40.962305+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T03:18:40.962Z" }, { "sequence": 1858, "attempt_id": "7b9bf8a0-2957-41fd-8bd6-d212f037475f", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.681715608923696, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:41.944576+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:18:41.944Z" }, { "sequence": 1859, "attempt_id": "974d655b-8d48-444c-8c8f-4655b443b630", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.166149959084578, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:18:42.428856+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:18:42.429Z" }, { "sequence": 1860, "attempt_id": "4bd5e5e2-eef1-4161-85f4-ca94b10c62e4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005924, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.29929928900674, "model_name": "gemini-3.1-pro-preview", "output_tokens": 303, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:42.522952+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005924, "created_at": "2026-09-17T03:18:42.523Z" }, { "sequence": 1861, "attempt_id": "08b5d65f-ea91-4af0-8c56-c41e1ea95e76", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0060115, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.5162623560754582, "model_name": "muse-spark-1.2", "output_tokens": 668, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:42.613869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0060115, "created_at": "2026-09-17T03:18:42.614Z" }, { "sequence": 1862, "attempt_id": "024e847e-0520-4a63-acdf-df476409536b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00573, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.936040922999382, "model_name": "gpt-5.6-terra", "output_tokens": 67, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:42.834193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00573, "created_at": "2026-09-17T03:18:42.834Z" }, { "sequence": 1863, "attempt_id": "37fdd431-8703-4fa2-b949-001bc47d2bd6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.446005166042596, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:42.960575+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:18:42.960Z" }, { "sequence": 1864, "attempt_id": "33eefefd-1e52-491c-b34f-ef28dd67a759", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01745, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.3685999719891697, "model_name": "claude-opus-5", "output_tokens": 108, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:43.050617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01745, "created_at": "2026-09-17T03:18:43.050Z" }, { "sequence": 1865, "attempt_id": "c49dbb9a-1f08-4faf-9e19-3bc7e7afdedc", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005862, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.1617020110134035, "model_name": "gpt-5.6-luna", "output_tokens": 78, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:43.213962+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005862, "created_at": "2026-09-17T03:18:43.214Z" }, { "sequence": 1866, "attempt_id": "1e3a3240-c774-4cc8-aa18-2377c7fb2a99", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.38589277502615, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:18:44.000072+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:18:44.000Z" }, { "sequence": 1867, "attempt_id": "03ca7ae0-7b97-423c-a8f1-971f82da7cc7", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0022425, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.6923715319717303, "model_name": "gemini-3.8-flash", "output_tokens": 356, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:18:44.215586+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0022425, "created_at": "2026-09-17T03:18:44.215Z" }, { "sequence": 1868, "attempt_id": "d297a128-e10a-45e6-8dba-432df28b34d1", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00884, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 4.751874592038803, "model_name": "claude-sonnet-5", "output_tokens": 294, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:44.520481+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00884, "created_at": "2026-09-17T03:18:44.520Z" }, { "sequence": 1869, "attempt_id": "c576e86f-26e9-4cf2-a8e7-97f6a060f561", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0053655, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 5.679681206005625, "model_name": "muse-spark-1.3", "output_tokens": 516, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:44.624037+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0053655, "created_at": "2026-09-17T03:18:44.624Z" }, { "sequence": 1870, "attempt_id": "6549ca3c-8e11-4d14-b45a-e0c7c706d679", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.6227974928915501, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:44.718507+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001876, "created_at": "2026-09-17T03:18:44.718Z" }, { "sequence": 1871, "attempt_id": "0751c114-7d16-4850-b9be-2f426d06a35b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005672, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.3785071560414508, "model_name": "gpt-5.6-luna", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:46.186507+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005672, "created_at": "2026-09-17T03:18:46.186Z" }, { "sequence": 1872, "attempt_id": "dc5e70f7-fb55-40b4-a3b7-c079a87e953f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00566, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.7702140850014985, "model_name": "gpt-5.6-terra", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:46.485173+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00566, "created_at": "2026-09-17T03:18:46.485Z" }, { "sequence": 1873, "attempt_id": "43175770-759a-45dd-8749-c6c0affafcf4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02589, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.26736388693098, "model_name": "gpt-6-astra", "output_tokens": 37, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:46.577648+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02589, "created_at": "2026-09-17T03:18:46.577Z" }, { "sequence": 1874, "attempt_id": "2794c891-4edf-41ea-9543-059e897f74d9", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017495, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.7770229120505974, "model_name": "claude-opus-5", "output_tokens": 131, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:46.917743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017495, "created_at": "2026-09-17T03:18:46.917Z" }, { "sequence": 1875, "attempt_id": "8d152738-a4f1-4ed8-bca1-658f67865944", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009278, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 4.027472860063426, "model_name": "claude-sonnet-5", "output_tokens": 359, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:47.331999+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009278, "created_at": "2026-09-17T03:18:47.332Z" }, { "sequence": 1876, "attempt_id": "f79b3e41-e3c6-41ea-8f9b-ddcca52caf37", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011276, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.829498266102746, "model_name": "gpt-5.6-sol", "output_tokens": 83, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:47.453776+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011276, "created_at": "2026-09-17T03:18:47.453Z" }, { "sequence": 1877, "attempt_id": "5dfdc7b1-fa21-4777-ab0b-aae05bbf5db4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 5.466734130983241, "model_name": "gemini-3.1-pro-preview", "output_tokens": 415, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:18:47.544838+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074, "created_at": "2026-09-17T03:18:47.545Z" }, { "sequence": 1878, "attempt_id": "99e2f555-393a-4641-8da8-037fa1c8b5b9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.8809923499356955, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:47.639222+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000374, "created_at": "2026-09-17T03:18:47.639Z" }, { "sequence": 1879, "attempt_id": "46bcf131-4dec-40a5-9a57-12bbe2df197a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03866, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 5.484394251019694, "model_name": "claude-fable-5-1", "output_tokens": 204, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:48.535350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03866, "created_at": "2026-09-17T03:18:48.535Z" }, { "sequence": 1880, "attempt_id": "b6a08cc4-2544-434e-8845-2d0c6c0a7e86", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077995, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 5.970136805088259, "model_name": "muse-spark-1.2", "output_tokens": 1069, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:48.898191+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0077995, "created_at": "2026-09-17T03:18:48.898Z" }, { "sequence": 1881, "attempt_id": "95c98eca-7fa9-4aa7-b6c0-83ab44ae03ef", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.2581240859581158, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:48.992059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005758, "created_at": "2026-09-17T03:18:48.992Z" }, { "sequence": 1882, "attempt_id": "6f0c3fe6-42bb-43bf-b018-8d21c97fc637", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.7913193010026589, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:49.434044+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T03:18:49.434Z" }, { "sequence": 1883, "attempt_id": "79720277-b82a-4ce6-8059-7636fe540fa8", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.031567556085065, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:49.671083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014395, "created_at": "2026-09-17T03:18:49.671Z" }, { "sequence": 1884, "attempt_id": "36eac01e-b4b7-4e2a-8512-3ca2b485335f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00452625, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 3.576905817957595, "model_name": "gemini-3.8-flash", "output_tokens": 976, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:18:50.154843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00452625, "created_at": "2026-09-17T03:18:50.155Z" }, { "sequence": 1885, "attempt_id": "7dc43366-90e3-4a01-b1b6-aaf875630e5e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.388984187040478, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:50.472736+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:18:50.472Z" }, { "sequence": 1886, "attempt_id": "95efd5fd-c927-4a89-9d9a-43581b8ccd9e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.8589964240090922, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:50.851327+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:18:50.851Z" }, { "sequence": 1887, "attempt_id": "2575b9db-0e46-41dd-98eb-f7e2c9492db5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00544, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.63019905006513, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:51.157859+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00544, "created_at": "2026-09-17T03:18:51.157Z" }, { "sequence": 1888, "attempt_id": "90fb26a8-b013-4f76-aded-7c76d90a6d35", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.703386669047177, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:51.251655+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T03:18:51.251Z" }, { "sequence": 1889, "attempt_id": "8d527972-e905-46ce-b415-2b0d9361ae62", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.7750284330686554, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:51.538144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:18:51.538Z" }, { "sequence": 1890, "attempt_id": "e94b591c-1783-4f03-b59b-76e20a30c046", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007094, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 8.926560633000918, "model_name": "muse-spark-1.3", "output_tokens": 903, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:51.634582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007094, "created_at": "2026-09-17T03:18:51.634Z" }, { "sequence": 1891, "attempt_id": "dde1f00c-ed51-45e8-80d5-34abc7a161a6", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00145425, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.2699464949546382, "model_name": "gemini-3.8-flash", "output_tokens": 159, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:51.836535+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00145425, "created_at": "2026-09-17T03:18:51.836Z" }, { "sequence": 1892, "attempt_id": "423d788a-3a7d-4caa-b1f5-4c526e8ecae3", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.347909044008702, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:52.292982+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003707, "created_at": "2026-09-17T03:18:52.293Z" }, { "sequence": 1893, "attempt_id": "281f7885-1168-41fa-9007-a86043e9ceaf", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00438375, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.198025421006605, "model_name": "muse-spark-1.2", "output_tokens": 285, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:53.544822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00438375, "created_at": "2026-09-17T03:18:53.545Z" }, { "sequence": 1894, "attempt_id": "026f4517-74eb-4979-985e-2b2a67e8f3d6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 1.2700057619949803, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:53.660233+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T03:18:53.660Z" }, { "sequence": 1895, "attempt_id": "eaeeb1eb-de9e-490b-b17a-2c08ba0504a2", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.8077237290563062, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:53.751850+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:18:53.753Z" }, { "sequence": 1896, "attempt_id": "8d8c199e-e148-465a-ba70-48211e9b3c5f", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.134305367944762, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:53.861867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T03:18:53.861Z" }, { "sequence": 1897, "attempt_id": "82ddb8d6-702f-410b-9b82-a93ed538fc51", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0089995, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 6.704364825040102, "model_name": "muse-spark-1.2", "output_tokens": 1369, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:54.129591+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0089995, "created_at": "2026-09-17T03:18:54.129Z" }, { "sequence": 1898, "attempt_id": "48f87ad2-1f05-4e4a-9343-8f980baff7f3", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00566, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.16043438797351, "model_name": "gemini-3.1-pro-preview", "output_tokens": 281, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:54.411822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00566, "created_at": "2026-09-17T03:18:54.411Z" }, { "sequence": 1899, "attempt_id": "9e7d13b7-02d9-4930-a593-94ae4522474e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.118125506094657, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:54.752989+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T03:18:54.753Z" }, { "sequence": 1900, "attempt_id": "dd7a3fd8-8a08-4372-bb94-9c180938431a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.8259955930989236, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:54.847659+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:18:54.847Z" }, { "sequence": 1901, "attempt_id": "fba338e6-656c-4371-ad68-5b24c0557208", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.1539014999289066, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:54.998899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:18:54.999Z" }, { "sequence": 1902, "attempt_id": "999320f6-c577-4389-9f7c-c77bdf1fecdf", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.7462361629586667, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:55.388700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:18:55.388Z" }, { "sequence": 1903, "attempt_id": "5a61d784-e304-4dd6-88a5-3bc90354d802", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.8460418169852346, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:18:55.693980+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:18:55.694Z" }, { "sequence": 1904, "attempt_id": "1b9a51d4-7a4e-4fe7-9e13-5d4e7c0c5402", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 2.048129120026715, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:55.800221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:18:55.800Z" }, { "sequence": 1905, "attempt_id": "df98f134-83ef-4280-b55d-cdf76c4fa2ef", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125175, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.4379473999142647, "model_name": "gemini-3.8-flash", "output_tokens": 111, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:55.943453+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00125175, "created_at": "2026-09-17T03:18:55.943Z" }, { "sequence": 1906, "attempt_id": "78aa0b9d-84be-439f-8f18-0ce2c13a0d91", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051785, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 5.2100804060464725, "model_name": "muse-spark-1.3", "output_tokens": 472, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:56.462035+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0051785, "created_at": "2026-09-17T03:18:56.462Z" }, { "sequence": 1907, "attempt_id": "fc3af050-0cd5-4d51-ae3f-8cc7a7e0434f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0039255, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.4634585860185325, "model_name": "muse-spark-1.2", "output_tokens": 186, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:56.558276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0039255, "created_at": "2026-09-17T03:18:56.558Z" }, { "sequence": 1908, "attempt_id": "cf2b426b-d006-4a17-8b77-bbfff1815c2c", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.7794589600525796, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:56.815040+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T03:18:56.815Z" }, { "sequence": 1909, "attempt_id": "b832b4a0-5fe2-48b3-919a-d79c758d244e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006188, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.053793561994098, "model_name": "claude-sonnet-5", "output_tokens": 56, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:57.946608+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006188, "created_at": "2026-09-17T03:18:57.946Z" }, { "sequence": 1910, "attempt_id": "da1ea077-c688-42b7-8316-71d20fb5266b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.4100929280975834, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:58.061709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:18:58.061Z" }, { "sequence": 1911, "attempt_id": "7cea6c09-37d2-482c-984b-f551534f34c9", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.8250469589838758, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:58.383651+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:18:58.383Z" }, { "sequence": 1912, "attempt_id": "dfb98fb4-b517-41d9-8f6d-cee4681124ec", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00554, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 4.193156302906573, "model_name": "gemini-3.1-pro-preview", "output_tokens": 276, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:58.485327+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00554, "created_at": "2026-09-17T03:18:58.486Z" }, { "sequence": 1913, "attempt_id": "c0d48d9a-45c7-4514-90a6-21d23b6343d2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005474, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.605654745013453, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:58.580345+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005474, "created_at": "2026-09-17T03:18:58.580Z" }, { "sequence": 1914, "attempt_id": "bf335cee-1f8f-4f24-8439-fe32e15ce28c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00447375, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.596363215940073, "model_name": "muse-spark-1.3", "output_tokens": 315, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:58.682921+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00447375, "created_at": "2026-09-17T03:18:58.683Z" }, { "sequence": 1915, "attempt_id": "8d1cd740-4fc1-412a-95a2-8c0753cd3273", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01547, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.998409310937859, "model_name": "claude-opus-5", "output_tokens": 56, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:58.787326+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01547, "created_at": "2026-09-17T03:18:58.787Z" }, { "sequence": 1916, "attempt_id": "4f423fc4-00bd-488d-963f-7a163d433b8a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.9463105360046029, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:58.993136+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:18:58.993Z" }, { "sequence": 1917, "attempt_id": "c2488b3f-72ba-450c-95ef-6f9f9ceeeddc", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00869775, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 12.015884542022832, "model_name": "muse-spark-1.3", "output_tokens": 1298, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:59.088116+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00869775, "created_at": "2026-09-17T03:18:59.089Z" }, { "sequence": 1918, "attempt_id": "46778cde-8733-49f8-b63d-3adcf0eddfed", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.60788536304608, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:59.182204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T03:18:59.182Z" }, { "sequence": 1919, "attempt_id": "0a68e611-3321-4983-8a23-83b91219f079", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035580000000000003, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.8060680039925501, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:59.386679+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00035580000000000003, "created_at": "2026-09-17T03:18:59.386Z" }, { "sequence": 1920, "attempt_id": "5a335387-1243-4ab8-9e2a-41d13b42865b", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.023154, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 14.085953654022887, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1737, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:00.367872+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.023154, "created_at": "2026-09-17T03:19:00.368Z" }, { "sequence": 1921, "attempt_id": "a277a03c-e14b-4153-8bcc-463d98a03215", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 1.1171778429998085, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:00.466723+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T03:19:00.468Z" }, { "sequence": 1922, "attempt_id": "c0d74690-9ac9-4efb-b3a9-66d5b249a6e6", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.7694852019194514, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:00.951977+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00601, "created_at": "2026-09-17T03:19:00.952Z" }, { "sequence": 1923, "attempt_id": "9cbbd3dc-4aed-4de2-a520-7c2e4ac09aa6", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00222075, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.5689682610100135, "model_name": "gemini-3.8-flash", "output_tokens": 365, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:01.058025+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00222075, "created_at": "2026-09-17T03:19:01.058Z" }, { "sequence": 1924, "attempt_id": "8b75b53d-a02b-4d31-a641-7c48ee76c658", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.9849438449600711, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:01.153135+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015025, "created_at": "2026-09-17T03:19:01.153Z" }, { "sequence": 1925, "attempt_id": "9ebbf47b-84b8-4dca-8366-208296f06bc3", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.343282360001467, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:01.824081+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:19:01.824Z" }, { "sequence": 1926, "attempt_id": "5e1b3de7-6da6-4399-a11d-e02396af6007", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005406, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.367348558967933, "model_name": "gpt-5.6-terra", "output_tokens": 40, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:01.928699+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005406, "created_at": "2026-09-17T03:19:01.928Z" }, { "sequence": 1927, "attempt_id": "b560d0c1-2ec4-4d3e-b107-ee3baa746365", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00554675, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.4089005099376664, "model_name": "muse-spark-1.2", "output_tokens": 561, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:02.186842+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00554675, "created_at": "2026-09-17T03:19:02.187Z" }, { "sequence": 1928, "attempt_id": "3e0bc3b7-ef46-4e83-ab72-793eb06703ad", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.972695626085624, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:02.439686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:19:02.439Z" }, { "sequence": 1929, "attempt_id": "89d05acf-7179-4b2b-8ab9-e9f152a0e95b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.644655196927488, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:02.534502+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:19:02.535Z" }, { "sequence": 1930, "attempt_id": "35e46449-685d-419e-b9c7-67957bce3bac", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006388, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.525633545010351, "model_name": "gemini-3.1-pro-preview", "output_tokens": 343, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:02.680295+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006388, "created_at": "2026-09-17T03:19:02.680Z" }, { "sequence": 1931, "attempt_id": "351dcf77-05e1-4d27-8d50-9c3b0c11b7df", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0050665, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 4.456694120075554, "model_name": "muse-spark-1.3", "output_tokens": 448, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:03.232286+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0050665, "created_at": "2026-09-17T03:19:03.232Z" }, { "sequence": 1932, "attempt_id": "7ecdf771-60c3-439a-a4b2-1f1890b1e0eb", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002115, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 2.103004071977921, "model_name": "gemini-3.8-flash", "output_tokens": 322, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:03.361477+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002115, "created_at": "2026-09-17T03:19:03.361Z" }, { "sequence": 1933, "attempt_id": "b5f146da-6547-4ebe-b89b-4a0d0c9a0350", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005633999999999999, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.4582583799492568, "model_name": "gpt-5.6-luna", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:03.516548+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005633999999999999, "created_at": "2026-09-17T03:19:03.516Z" }, { "sequence": 1934, "attempt_id": "ca86a8ae-447b-4cce-b04a-ee770338697e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.6223355419933796, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:03.614142+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:19:03.614Z" }, { "sequence": 1935, "attempt_id": "887f29ec-4c89-436c-b545-061a9039ea7f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.6277407599845901, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:04.402073+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00601, "created_at": "2026-09-17T03:19:04.402Z" }, { "sequence": 1936, "attempt_id": "82408e73-b2a2-4438-ad9b-0c084563c565", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 1.1583858800586313, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:04.512553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T03:19:04.513Z" }, { "sequence": 1937, "attempt_id": "618311fa-5681-4a04-ae67-3b6fa3b4684b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.9901694020954892, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:04.618246+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015025, "created_at": "2026-09-17T03:19:04.618Z" }, { "sequence": 1938, "attempt_id": "522227bd-c2a9-4203-853e-3b59cab462b8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.2556929659331217, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:04.870129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:19:04.870Z" }, { "sequence": 1939, "attempt_id": "f73847f3-d4d8-4257-a281-389874d2246d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00543, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.3240930089959875, "model_name": "gpt-5.6-terra", "output_tokens": 42, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:05.036208+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00543, "created_at": "2026-09-17T03:19:05.036Z" }, { "sequence": 1940, "attempt_id": "97e4f46c-1ce6-4d6c-bd97-38bd1665e51e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.9407228549243882, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:19:05.906759+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:19:05.906Z" }, { "sequence": 1941, "attempt_id": "8df445bf-ba7c-45fc-bcac-dc16a7ac1043", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.398761176969856, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:06.007341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:19:06.007Z" }, { "sequence": 1942, "attempt_id": "e1d19a12-bf26-4f11-a5e2-59d6aca82146", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.8091006990289316, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:06.264854+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:19:06.265Z" }, { "sequence": 1943, "attempt_id": "834d68c8-7880-4344-a6f5-61cdb68b9ef6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0058785, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.086923344992101, "model_name": "muse-spark-1.2", "output_tokens": 617, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:06.369704+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0058785, "created_at": "2026-09-17T03:19:06.369Z" }, { "sequence": 1944, "attempt_id": "b03d99a3-5af2-48a4-9114-81f4c5d0bf1c", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00176625, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.7102847839705646, "model_name": "gemini-3.8-flash", "output_tokens": 229, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:19:06.464088+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00176625, "created_at": "2026-09-17T03:19:06.465Z" }, { "sequence": 1945, "attempt_id": "831b5d6e-b8dd-423b-8aa8-4dfef1bb4b0c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004448, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 5.4734241019468755, "model_name": "gemini-3.1-pro-preview", "output_tokens": 169, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:06.626825+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004448, "created_at": "2026-09-17T03:19:06.626Z" }, { "sequence": 1946, "attempt_id": "3d0a7caa-9ef5-47b3-83e4-706f673e1293", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005658000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.328513108077459, "model_name": "gpt-5.6-luna", "output_tokens": 61, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:06.841348+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005658000000000001, "created_at": "2026-09-17T03:19:06.841Z" }, { "sequence": 1947, "attempt_id": "47e81a34-3a73-4720-b516-421c956fe802", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.7448666499694809, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:07.303418+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T03:19:07.303Z" }, { "sequence": 1948, "attempt_id": "d7109301-b4c0-4e93-9023-2a1fad8c8229", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00504125, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.09617640695069, "model_name": "muse-spark-1.2", "output_tokens": 420, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:08.103833+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00504125, "created_at": "2026-09-17T03:19:08.104Z" }, { "sequence": 1949, "attempt_id": "3003ada6-16a9-4b72-b80d-5b8e0ca70d22", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.7354092119494453, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:08.203366+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005736, "created_at": "2026-09-17T03:19:08.204Z" }, { "sequence": 1950, "attempt_id": "8b99155a-8087-4675-b698-6ea00401d980", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.343289625016041, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:08.300345+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:19:08.300Z" }, { "sequence": 1951, "attempt_id": "44ca173b-1ad2-4b9a-805e-2c3f00edc1b6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.7188507179962471, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:08.454057+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:19:08.454Z" }, { "sequence": 1952, "attempt_id": "ff4519d3-dd12-4a9b-88a5-58aef3836687", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005398, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.3755663969786838, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:08.777531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005398, "created_at": "2026-09-17T03:19:08.777Z" }, { "sequence": 1953, "attempt_id": "a25dcb69-2e97-41f3-9747-70e4f56abf1d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005756, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.189881289028563, "model_name": "gemini-3.1-pro-preview", "output_tokens": 278, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:08.875944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005756, "created_at": "2026-09-17T03:19:08.877Z" }, { "sequence": 1954, "attempt_id": "b93ecc69-3de4-4caf-be1e-15c8f56e4d58", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 0.9079597661038861, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:09.111591+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:19:09.111Z" }, { "sequence": 1955, "attempt_id": "abe76c1d-7e40-42ee-a5b0-2775024d767b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01544, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.7679634500527754, "model_name": "claude-opus-5", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:09.208514+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01544, "created_at": "2026-09-17T03:19:09.209Z" }, { "sequence": 1956, "attempt_id": "455fad39-d679-4213-8a44-4ef327d13621", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-002", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.7777792930137366, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:19:09.328361+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:19:09.328Z" }, { "sequence": 1957, "attempt_id": "f5eaec37-d466-4018-af51-fefcbfcdf6b0", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.423034635023214, "model_name": "muse-spark-1.3", "output_tokens": 505, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:09.554965+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0054025, "created_at": "2026-09-17T03:19:09.555Z" }, { "sequence": 1958, "attempt_id": "8c1ea14f-a1a5-4692-a4c5-e78beb58ca02", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001575, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 2.0698981600580737, "model_name": "gemini-3.8-flash", "output_tokens": 192, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:10.464839+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001575, "created_at": "2026-09-17T03:19:10.465Z" }, { "sequence": 1959, "attempt_id": "1d41caef-f545-4d79-b06e-4d3bb129cf37", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.2050374679965898, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:10.857310+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:19:10.857Z" }, { "sequence": 1960, "attempt_id": "7ae4e905-01d1-4de8-8584-e7678fc4c133", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 4.834709201939404, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:10.958188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T03:19:10.958Z" }, { "sequence": 1961, "attempt_id": "d101460c-3f11-4fe4-81a3-388e3b615632", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6420017270138487, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:11.067395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:19:11.067Z" }, { "sequence": 1962, "attempt_id": "37577f0a-822e-4a70-acf2-216a350b4b45", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00675825, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 9.42623075901065, "model_name": "muse-spark-1.3", "output_tokens": 824, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:11.448912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00675825, "created_at": "2026-09-17T03:19:11.449Z" }, { "sequence": 1963, "attempt_id": "8d34a5c8-d72f-4dc8-b507-7740eeacc5c8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004941, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 2.7518417260143906, "model_name": "muse-spark-1.2", "output_tokens": 417, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:11.723545+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004941, "created_at": "2026-09-17T03:19:11.723Z" }, { "sequence": 1964, "attempt_id": "b4a49e9d-d094-4fa2-b611-ce13e5ecde0d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0571022579679266, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:12.111425+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:19:12.111Z" }, { "sequence": 1965, "attempt_id": "d4e283af-13a8-47fb-901f-9878bf173250", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005076, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.990112956962548, "model_name": "gemini-3.1-pro-preview", "output_tokens": 233, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:12.290708+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005076, "created_at": "2026-09-17T03:19:12.290Z" }, { "sequence": 1966, "attempt_id": "65ef6df9-a41b-4e03-b37e-1c7c6fa727d5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.383316284045577, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:12.386741+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:19:12.388Z" }, { "sequence": 1967, "attempt_id": "d7577e50-5ed7-4617-9c41-d3dfb541d002", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.172266007051803, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:12.493664+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:19:12.493Z" }, { "sequence": 1968, "attempt_id": "5a2e679d-5e31-457b-926f-797ed9d9fd3b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.4470467569772154, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:13.012629+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:19:13.012Z" }, { "sequence": 1969, "attempt_id": "f3c9618c-0041-4004-ab63-67ca351df30c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.859576813993044, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:13.113222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:19:13.114Z" }, { "sequence": 1970, "attempt_id": "ff93b410-6a6e-400f-8ffd-22306cb45aa0", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8193403850309551, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:13.207507+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:19:13.207Z" }, { "sequence": 1971, "attempt_id": "c83c7f58-8047-4d80-8cb4-a0c7de230f53", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.0603672560537234, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:13.306039+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:19:13.306Z" }, { "sequence": 1972, "attempt_id": "18c54f7f-552a-4f4a-a40c-3ce1c40f8372", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.6942421710118651, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:14.000577+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:19:14.000Z" }, { "sequence": 1973, "attempt_id": "ce358d74-04bb-479c-8432-d901dbaa88a8", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129525, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.506996867014095, "model_name": "gemini-3.8-flash", "output_tokens": 118, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:14.327419+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00129525, "created_at": "2026-09-17T03:19:14.327Z" }, { "sequence": 1974, "attempt_id": "98279b9f-e0ae-4d0c-a81b-d73920fe99e8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004137, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.6075911769876257, "model_name": "muse-spark-1.3", "output_tokens": 229, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:14.994636+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004137, "created_at": "2026-09-17T03:19:14.994Z" }, { "sequence": 1975, "attempt_id": "82ba66cd-c812-4244-a33e-0c58eb1831cc", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010144, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.1543746460229158, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:15.256744+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010144, "created_at": "2026-09-17T03:19:15.256Z" }, { "sequence": 1976, "attempt_id": "4332ea3b-cb01-434a-b3ac-ffe3e63f94d2", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003354, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.875242267968133, "model_name": "gemini-3.1-pro-preview", "output_tokens": 90, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:15.423706+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003354, "created_at": "2026-09-17T03:19:15.423Z" }, { "sequence": 1977, "attempt_id": "1d4bfad9-9829-4a66-ae48-3699f30ebc75", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02641, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.127089389017783, "model_name": "gpt-6-astra", "output_tokens": 35, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:15.534349+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02641, "created_at": "2026-09-17T03:19:15.534Z" }, { "sequence": 1978, "attempt_id": "4803db15-af7d-4b27-b025-3a9f99c17962", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007158, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.6904853919986635, "model_name": "claude-sonnet-5", "output_tokens": 129, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:15.994208+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007158, "created_at": "2026-09-17T03:19:15.994Z" }, { "sequence": 1979, "attempt_id": "06a49705-5428-4589-9b05-2a84a865dd30", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.553407175000757, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:16.143045+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:19:16.143Z" }, { "sequence": 1980, "attempt_id": "00bfd885-f788-4da9-85ee-38028830f6fe", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005448, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.765558535931632, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:16.239677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005448, "created_at": "2026-09-17T03:19:16.240Z" }, { "sequence": 1981, "attempt_id": "01bf962b-abde-40d9-bf61-d569e183953d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0053865, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.799622353981249, "model_name": "muse-spark-1.2", "output_tokens": 523, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:16.338092+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0053865, "created_at": "2026-09-17T03:19:16.339Z" }, { "sequence": 1982, "attempt_id": "1e2ee58d-e5bc-4b8d-9ff5-996717485e3f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00603325, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 7.416966984979808, "model_name": "muse-spark-1.3", "output_tokens": 674, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:16.433655+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00603325, "created_at": "2026-09-17T03:19:16.433Z" }, { "sequence": 1983, "attempt_id": "13d79aae-1529-4b2b-b47c-7e37b36f9979", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01757, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.2535108840093017, "model_name": "claude-opus-5", "output_tokens": 116, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:16.531031+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01757, "created_at": "2026-09-17T03:19:16.531Z" }, { "sequence": 1984, "attempt_id": "601c90fd-909b-44bd-bd4d-65d3f17e8bcf", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.9410715179983526, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:19:16.628590+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:19:16.630Z" }, { "sequence": 1985, "attempt_id": "53fc146f-32be-420f-8043-e171440a0738", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001851, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.691694593988359, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "A" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:17.320601+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001851, "created_at": "2026-09-17T03:19:17.320Z" }, { "sequence": 1986, "attempt_id": "cdd285c5-3df3-495b-b825-2cbab14f6741", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00142725, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.588262823992409, "model_name": "gemini-3.8-flash", "output_tokens": 138, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:18.122908+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00142725, "created_at": "2026-09-17T03:19:18.123Z" }, { "sequence": 1987, "attempt_id": "8105de83-3bd0-4bf0-b5d5-bca0ae9a83b8", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.006242, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.0450759700033814, "model_name": "claude-sonnet-5", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:18.576390+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006242, "created_at": "2026-09-17T03:19:18.576Z" }, { "sequence": 1988, "attempt_id": "d8845ade-c4b7-482a-b72e-24d3a05d6e84", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.107137589016929, "model_name": "gpt-5.6-luna", "output_tokens": 89, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:19.204539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006, "created_at": "2026-09-17T03:19:19.204Z" }, { "sequence": 1989, "attempt_id": "a6191436-978b-4cb3-9d2d-4d45b561be83", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02684, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.6291073210304603, "model_name": "gpt-6-astra", "output_tokens": 60, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:19.355897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02684, "created_at": "2026-09-17T03:19:19.356Z" }, { "sequence": 1990, "attempt_id": "88a21f02-8f01-4140-9b01-b78184cc8b5d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005572, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.064939368981868, "model_name": "gpt-5.6-terra", "output_tokens": 67, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:20.301353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005572, "created_at": "2026-09-17T03:19:20.301Z" }, { "sequence": 1991, "attempt_id": "a48c3e0c-6709-4f24-b9c1-0b42cd9454a6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005716, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7227470090147108, "model_name": "gpt-5.6-luna", "output_tokens": 79, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:20.405252+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005716, "created_at": "2026-09-17T03:19:20.405Z" }, { "sequence": 1992, "attempt_id": "4e8fa83d-d768-47ca-ab27-616e93362b23", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00713, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.066797427949496, "model_name": "gemini-3.1-pro-preview", "output_tokens": 392, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:20.506687+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00713, "created_at": "2026-09-17T03:19:20.508Z" }, { "sequence": 1993, "attempt_id": "c6e97f0f-557f-45cb-a29b-92ddf5dedd4c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.020355, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 4.701592938043177, "model_name": "claude-opus-5", "output_tokens": 249, "prediction": { "choice": "A" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:21.230148+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.020355, "created_at": "2026-09-17T03:19:21.230Z" }, { "sequence": 1994, "attempt_id": "13bd8d04-efdc-46f8-b7a1-d0c3439b7385", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02913, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.021841080044396, "model_name": "claude-fable-5-1", "output_tokens": 17, "prediction": { "choice": "B" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:21.360192+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02913, "created_at": "2026-09-17T03:19:21.360Z" }, { "sequence": 1995, "attempt_id": "34d68988-2ba8-4701-ad9e-158b85cdc9f7", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 1.1096744410460815, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:21.515192+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003668, "created_at": "2026-09-17T03:19:21.515Z" }, { "sequence": 1996, "attempt_id": "5e0e00a5-2716-48e6-b117-f323dd9e6203", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012016, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 4.373948159976862, "model_name": "gpt-5.6-sol", "output_tokens": 124, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:21.797127+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012016, "created_at": "2026-09-17T03:19:21.797Z" }, { "sequence": 1997, "attempt_id": "031a76ab-4bad-4d56-9e2c-21c784fd892e", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6311255840118974, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:22.526970+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:19:22.527Z" }, { "sequence": 1998, "attempt_id": "db85d6fc-6c94-43f7-816b-d948e4074179", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.5140671629924327, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:23.126237+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:19:23.126Z" }, { "sequence": 1999, "attempt_id": "cdba931c-ec4b-49b4-b499-3101cc333d5d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00302325, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 3.8924536980921403, "model_name": "gemini-3.8-flash", "output_tokens": 580, "prediction": { "choice": "B" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:23.345545+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00302325, "created_at": "2026-09-17T03:19:23.345Z" }, { "sequence": 2000, "attempt_id": "ac4f48b5-ad8c-4677-8083-232e54f6dffc", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00904725, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 7.377113669062965, "model_name": "muse-spark-1.2", "output_tokens": 1362, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:23.617062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00904725, "created_at": "2026-09-17T03:19:23.617Z" }, { "sequence": 2001, "attempt_id": "4a9099f8-8d58-4d0a-9c89-fe8ac9b2301e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0215230480534956, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:24.465725+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:19:24.465Z" }, { "sequence": 2002, "attempt_id": "f6b4646c-d9ee-4b83-98e3-be187d49bbbe", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.367727342993021, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:24.595121+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:19:24.595Z" }, { "sequence": 2003, "attempt_id": "cb83498e-0576-4f0c-8522-a3eb8757128b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.39313070604112, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:24.725147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:19:24.725Z" }, { "sequence": 2004, "attempt_id": "2ecbf3f2-deff-4550-8939-9ae8eba75e39", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00861, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 5.665666086948477, "model_name": "gemini-3.1-pro-preview", "output_tokens": 529, "prediction": { "choice": "B" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:24.970982+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00861, "created_at": "2026-09-17T03:19:24.971Z" }, { "sequence": 2005, "attempt_id": "db63beca-8296-4d77-a272-a06814dbb803", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 3.5674801979912445, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:25.071518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:19:25.072Z" }, { "sequence": 2006, "attempt_id": "1279212f-242b-4793-9190-50e2afe29e6c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.2717818020610139, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:25.169271+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:19:25.169Z" }, { "sequence": 2007, "attempt_id": "25ceed01-1977-46f1-9a1e-39f3e9dfafbb", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.746525185997598, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:25.387469+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:19:25.387Z" }, { "sequence": 2008, "attempt_id": "02882fa8-430f-4560-a458-5f00327d815f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.053692044923082, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:25.876041+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:19:25.876Z" }, { "sequence": 2009, "attempt_id": "80073f45-9ad4-4268-ab94-070c3eaa5bff", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001278, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.513899810030125, "model_name": "gemini-3.8-flash", "output_tokens": 114, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:26.206371+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001278, "created_at": "2026-09-17T03:19:26.206Z" }, { "sequence": 2010, "attempt_id": "2f4dc2d0-df10-4b7c-9e3a-0e9c3e542dbb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00396325, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.5218042549677193, "model_name": "muse-spark-1.2", "output_tokens": 189, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:26.787912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00396325, "created_at": "2026-09-17T03:19:26.788Z" }, { "sequence": 2011, "attempt_id": "dc2f9788-f82e-4825-ae25-1a7fdb15bf90", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6488368820864707, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:26.955521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:19:26.955Z" }, { "sequence": 2012, "attempt_id": "c34f2e33-d6cc-4f08-b321-bd2744470efe", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003576, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.8603506629588082, "model_name": "gemini-3.1-pro-preview", "output_tokens": 109, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:27.428654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003576, "created_at": "2026-09-17T03:19:27.428Z" }, { "sequence": 2013, "attempt_id": "35319c9c-f290-484d-be4e-bb6cd6b69885", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.2765339529141784, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:27.777031+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01447, "created_at": "2026-09-17T03:19:27.777Z" }, { "sequence": 2014, "attempt_id": "d282b695-3594-4f5c-bb5e-5f4dedcb7673", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006388, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.388393223984167, "model_name": "claude-sonnet-5", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:28.366528+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006388, "created_at": "2026-09-17T03:19:28.366Z" }, { "sequence": 2015, "attempt_id": "c9d98c8b-1ae0-44b8-9a95-783301765163", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6619773249840364, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:28.551768+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:19:28.551Z" }, { "sequence": 2016, "attempt_id": "b21e44a4-3350-4a99-959d-8ca0dbab04a8", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.380320427007973, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:28.653357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:19:28.653Z" }, { "sequence": 2017, "attempt_id": "a1a7e065-db7c-49a7-9141-52dc34259673", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.0443327160319313, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:28.921985+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004968, "created_at": "2026-09-17T03:19:28.922Z" }, { "sequence": 2018, "attempt_id": "35cd377c-c190-4d5d-b8d6-b3388d4f8936", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008771, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 12.844864701968618, "model_name": "muse-spark-1.3", "output_tokens": 1297, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:29.034950+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008771, "created_at": "2026-09-17T03:19:29.035Z" }, { "sequence": 2019, "attempt_id": "691aa402-002d-4450-81ca-48fefcd65492", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00457525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.965552462032065, "model_name": "muse-spark-1.3", "output_tokens": 333, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:29.133859+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00457525, "created_at": "2026-09-17T03:19:29.135Z" }, { "sequence": 2020, "attempt_id": "f7507d9f-a412-40d8-920c-1a8e18fbdd67", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7394243570743129, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:29.492136+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:19:29.492Z" }, { "sequence": 2021, "attempt_id": "553c2bcb-4f96-44e8-b8c5-8d5bd3188c23", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0016725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.5393984199035913, "model_name": "gemini-3.8-flash", "output_tokens": 216, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:19:30.193052+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0016725, "created_at": "2026-09-17T03:19:30.193Z" }, { "sequence": 2022, "attempt_id": "8687bdf1-8357-41ce-9c32-2df56ac6baf8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005496, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.813144402927719, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:30.346016+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005496, "created_at": "2026-09-17T03:19:30.346Z" }, { "sequence": 2023, "attempt_id": "9953873c-1049-40de-99f8-ee6dbe79fd78", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.4805178829701617, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:30.534743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:19:30.534Z" }, { "sequence": 2024, "attempt_id": "236e53c9-bcd0-4335-aabc-2c75234c369a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.6757539740065113, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:31.121341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T03:19:31.121Z" }, { "sequence": 2025, "attempt_id": "a364e961-bc40-4b61-a6cb-ba830817d491", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016045, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.6665223359595984, "model_name": "claude-opus-5", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:32.257847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016045, "created_at": "2026-09-17T03:19:32.258Z" }, { "sequence": 2026, "attempt_id": "8ef22892-4f9f-4234-af53-de499fbdd80a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.3072714139707386, "model_name": "claude-sonnet-5", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:32.602860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006458, "created_at": "2026-09-17T03:19:32.602Z" }, { "sequence": 2027, "attempt_id": "520fa5b1-1720-4d03-81c7-301bf82e636a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.0866045899456367, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:32.721974+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:19:32.722Z" }, { "sequence": 2028, "attempt_id": "afbcf0fe-e9d1-4736-8aca-eb0c3080b8b3", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.930145274964161, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:33.162311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T03:19:33.162Z" }, { "sequence": 2029, "attempt_id": "16d8672b-e882-4e1e-9887-58b68727f07e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00544575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.147739412961528, "model_name": "muse-spark-1.2", "output_tokens": 534, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:33.281845+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00544575, "created_at": "2026-09-17T03:19:33.281Z" }, { "sequence": 2030, "attempt_id": "1ff39da4-d6e0-4263-a2c9-53a5ba47d545", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010692, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.232412977027707, "model_name": "gpt-5.6-sol", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:33.456101+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010692, "created_at": "2026-09-17T03:19:33.456Z" }, { "sequence": 2031, "attempt_id": "5135caef-ccad-470e-a593-2f34daae8d23", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.381106938002631, "model_name": "gpt-5.6-terra", "output_tokens": 52, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:33.742693+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0054, "created_at": "2026-09-17T03:19:33.742Z" }, { "sequence": 2032, "attempt_id": "df37f85f-5ec0-46db-acac-21cfc9fcb549", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051185, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.803880111081526, "model_name": "muse-spark-1.3", "output_tokens": 457, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:33.843642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0051185, "created_at": "2026-09-17T03:19:33.843Z" }, { "sequence": 2033, "attempt_id": "a8959fca-ca9c-4740-82d4-fa1e3b0d7d6e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.8453296589432284, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:34.224855+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000368, "created_at": "2026-09-17T03:19:34.224Z" }, { "sequence": 2034, "attempt_id": "8cf0bd6b-15a0-4c90-a728-9994a80acc7c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00962975, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 14.277560935006477, "model_name": "muse-spark-1.3", "output_tokens": 1522, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:34.784529+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00962975, "created_at": "2026-09-17T03:19:34.784Z" }, { "sequence": 2035, "attempt_id": "21525222-f641-4f57-921e-6cea7142cfa3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.43936117796693, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:35.142824+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004944, "created_at": "2026-09-17T03:19:35.142Z" }, { "sequence": 2036, "attempt_id": "9a335b98-abbb-401a-9db2-66626b7fc5eb", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.7168398959329352, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:35.960907+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T03:19:35.961Z" }, { "sequence": 2037, "attempt_id": "a26cf410-1a51-4735-9792-1bff1f7bf875", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00339, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 2.810856053023599, "model_name": "gemini-3.8-flash", "output_tokens": 677, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:36.075583+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00339, "created_at": "2026-09-17T03:19:36.075Z" }, { "sequence": 2038, "attempt_id": "d0d39f03-1aee-4b15-89d8-4ea54399cb14", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01226, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 8.402571517042816, "model_name": "gemini-3.1-pro-preview", "output_tokens": 830, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:36.869960+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01226, "created_at": "2026-09-17T03:19:36.870Z" }, { "sequence": 2039, "attempt_id": "4f35a800-5e4f-43ad-9a80-308e5cf295d1", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006398, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.410622815019451, "model_name": "claude-sonnet-5", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:37.296988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006398, "created_at": "2026-09-17T03:19:37.297Z" }, { "sequence": 2040, "attempt_id": "a361b689-8344-42ea-887b-35708b5c4eed", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00803, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.298401274019852, "model_name": "gemini-3.1-pro-preview", "output_tokens": 480, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:38.130211+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00803, "created_at": "2026-09-17T03:19:38.130Z" }, { "sequence": 2041, "attempt_id": "b1a710b8-ee5b-4bb1-b35e-f2362d26a3a7", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01892, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 4.038402513950132, "model_name": "claude-opus-5", "output_tokens": 192, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:38.364026+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01892, "created_at": "2026-09-17T03:19:38.364Z" }, { "sequence": 2042, "attempt_id": "7dc99623-fe96-434a-86f9-4378826d51eb", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 4.451809523976408, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:38.464886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T03:19:38.465Z" }, { "sequence": 2043, "attempt_id": "2109c5ec-59e5-400c-9d1c-a3ebff17925c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.9880034669768065, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:38.562101+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:19:38.562Z" }, { "sequence": 2044, "attempt_id": "f675fe2e-7ca1-4198-9b54-f672f246d0ef", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.680896883015521, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:38.855714+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:19:38.855Z" }, { "sequence": 2045, "attempt_id": "c7622392-ddc8-48de-9c31-27900e10b540", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 3.042258986039087, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:39.107428+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:19:39.107Z" }, { "sequence": 2046, "attempt_id": "21e09689-5186-464f-a628-72090b44d64e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.3587010781047866, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:40.021331+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003707, "created_at": "2026-09-17T03:19:40.021Z" }, { "sequence": 2047, "attempt_id": "5431e0c0-abc1-43de-97c9-9595f2a51a86", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00538, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 3.182056565070525, "model_name": "gpt-5.6-terra", "output_tokens": 49, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:40.167361+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00538, "created_at": "2026-09-17T03:19:40.167Z" }, { "sequence": 2048, "attempt_id": "5142b8b2-0923-44f7-ab26-df1ff7551016", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00179175, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.978606307064183, "model_name": "gemini-3.8-flash", "output_tokens": 249, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:40.443786+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00179175, "created_at": "2026-09-17T03:19:40.443Z" }, { "sequence": 2049, "attempt_id": "bbee5cde-2cc7-45c5-b67b-8dc848cc7ece", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.206489242031239, "model_name": "muse-spark-1.2", "output_tokens": 350, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:41.163548+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00466, "created_at": "2026-09-17T03:19:41.163Z" }, { "sequence": 2050, "attempt_id": "3d4eed3b-a60e-458a-936e-0c1a77f69f39", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.7060911030275747, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:41.268670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:19:41.268Z" }, { "sequence": 2051, "attempt_id": "e1758252-231e-4934-aa84-3e9bda34e533", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.5548049189383164, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:41.822912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:19:41.823Z" }, { "sequence": 2052, "attempt_id": "0bca3aa7-a1ad-4f7c-99ea-1fd6b73b5d7b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.050981136970222, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:42.177958+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:19:42.178Z" }, { "sequence": 2053, "attempt_id": "07335406-791f-4f9f-a57a-3da88666eead", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1497244889615104, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:42.519105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:19:42.519Z" }, { "sequence": 2054, "attempt_id": "cd3f7071-8f33-4013-8bc1-48883c349a16", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004136, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.386484388960525, "model_name": "gemini-3.1-pro-preview", "output_tokens": 154, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:42.624766+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004136, "created_at": "2026-09-17T03:19:42.624Z" }, { "sequence": 2055, "attempt_id": "7e572b61-4b96-44fd-8031-16652ff02e0d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.5819608939345926, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:42.790720+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:19:42.790Z" }, { "sequence": 2056, "attempt_id": "45cffcd1-bf86-4f20-abe8-42415c047683", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.167948403977789, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:43.094760+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:19:43.094Z" }, { "sequence": 2057, "attempt_id": "f18cc55c-7554-4aef-95bd-5371a8e7056b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.1516727240523323, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:43.420614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:19:43.420Z" }, { "sequence": 2058, "attempt_id": "155201d4-107e-444a-903f-0a1a05fc73d8", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.8978101019747555, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:43.789008+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:19:43.789Z" }, { "sequence": 2059, "attempt_id": "2b992665-c3c7-4365-8fff-fd86e098f6d9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.6183784080203623, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:43.904522+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:19:43.904Z" }, { "sequence": 2060, "attempt_id": "2c76b9b4-ad3d-48ac-8766-0204da17f0d8", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00128175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.630324814002961, "model_name": "gemini-3.8-flash", "output_tokens": 115, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:44.356667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00128175, "created_at": "2026-09-17T03:19:44.356Z" }, { "sequence": 2061, "attempt_id": "fdc6b6a8-4a20-418c-b1e2-b945de0f56fd", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0222735, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 23.933071587933227, "model_name": "muse-spark-1.2", "output_tokens": 4497, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:44.537187+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0222735, "created_at": "2026-09-17T03:19:44.537Z" }, { "sequence": 2062, "attempt_id": "7416d22e-8096-485b-b736-0d505aad24c1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00393775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.5903011920163408, "model_name": "muse-spark-1.3", "output_tokens": 183, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:44.786394+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00393775, "created_at": "2026-09-17T03:19:44.786Z" }, { "sequence": 2063, "attempt_id": "cf0f76a2-1794-4ae7-9d80-0c7f8990d6da", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.5449982489226386, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:45.182474+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001884, "created_at": "2026-09-17T03:19:45.182Z" }, { "sequence": 2064, "attempt_id": "1646591f-5856-40bb-9b77-58e357378dcc", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00410775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6827544840052724, "model_name": "muse-spark-1.2", "output_tokens": 223, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:45.286401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00410775, "created_at": "2026-09-17T03:19:45.286Z" }, { "sequence": 2065, "attempt_id": "f45ea816-fb20-4d69-ba17-8fc366e22345", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003456, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.6787800930906087, "model_name": "gemini-3.1-pro-preview", "output_tokens": 99, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:46.303867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003456, "created_at": "2026-09-17T03:19:46.304Z" }, { "sequence": 2066, "attempt_id": "8c1a89fc-4361-4df3-b414-67de7eb44c70", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005918, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 8.185101227019913, "model_name": "muse-spark-1.3", "output_tokens": 646, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:46.851049+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005918, "created_at": "2026-09-17T03:19:46.851Z" }, { "sequence": 2067, "attempt_id": "f5768b4e-01e5-4f6f-9379-351df3f0811f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006826, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 2.550000458024442, "model_name": "claude-sonnet-5", "output_tokens": 109, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:47.010020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006826, "created_at": "2026-09-17T03:19:47.010Z" }, { "sequence": 2068, "attempt_id": "ab016a9e-bf83-46f2-88c1-1f791838becb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005692, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.0180877830134705, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:47.407041+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005692, "created_at": "2026-09-17T03:19:47.407Z" }, { "sequence": 2069, "attempt_id": "bba0796b-b971-4f89-bcb1-2db69f223954", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004996, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.056236900971271, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:47.512958+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004996, "created_at": "2026-09-17T03:19:47.513Z" }, { "sequence": 2070, "attempt_id": "4d5f27c4-3df8-4be8-91ec-79265156e31a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02925, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 3.633980589103885, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:47.617395+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02925, "created_at": "2026-09-17T03:19:47.617Z" }, { "sequence": 2071, "attempt_id": "9f28aa33-c0fa-4814-930c-0a12957d1af7", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02624, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.7700190019095317, "model_name": "gpt-6-astra", "output_tokens": 42, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:47.719876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02624, "created_at": "2026-09-17T03:19:47.720Z" }, { "sequence": 2072, "attempt_id": "c4ceefb2-c3e6-4677-acb2-d03b4fe63bde", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.019215, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.81181441701483, "model_name": "claude-opus-5", "output_tokens": 195, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:47.822763+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.019215, "created_at": "2026-09-17T03:19:47.822Z" }, { "sequence": 2073, "attempt_id": "22b29d5c-ea41-481e-93f0-55e391ff3a9b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011196, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.6475736260181293, "model_name": "gpt-5.6-sol", "output_tokens": 77, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:47.934209+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011196, "created_at": "2026-09-17T03:19:47.934Z" }, { "sequence": 2074, "attempt_id": "e0fff9dd-6061-4822-9398-cc438405c1d5", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036359999999999995, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.8213586539495736, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:48.334597+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036359999999999995, "created_at": "2026-09-17T03:19:48.334Z" }, { "sequence": 2075, "attempt_id": "f84ac9e4-46e8-4cee-a8dd-b6018e80cef5", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.6871591559611261, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:49.127607+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T03:19:49.127Z" }, { "sequence": 2076, "attempt_id": "cbf24630-86a8-4cfa-823e-4aeaecd2699e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.7822195639600977, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:49.817121+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T03:19:49.817Z" }, { "sequence": 2077, "attempt_id": "a2f28ba2-5c08-4775-920a-ef57b089dee9", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.5284518400439993, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:50.454487+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T03:19:50.454Z" }, { "sequence": 2078, "attempt_id": "d31a62ed-71ed-443b-9715-5d67f34c15b5", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.7337882320862263, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:50.968831+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:19:50.968Z" }, { "sequence": 2079, "attempt_id": "c175c927-3e9b-4d84-b272-35e719d5c236", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.6062139560235664, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:51.429265+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T03:19:51.429Z" }, { "sequence": 2080, "attempt_id": "265252f0-feb4-4813-8083-3a4835aca84a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.0261568380519748, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:51.598670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:19:51.598Z" }, { "sequence": 2081, "attempt_id": "7aed1663-e7e3-4790-874b-9f65780ce09a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.716109007014893, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:51.701729+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:19:51.703Z" }, { "sequence": 2082, "attempt_id": "9a81cc99-4a99-4d78-aaa4-02b6da38e907", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3101414470002055, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:53.115868+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:19:53.116Z" }, { "sequence": 2083, "attempt_id": "6941947f-db87-43da-937f-ad7a1cd0ac63", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.181448891060427, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:53.256383+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:19:53.256Z" }, { "sequence": 2084, "attempt_id": "8f427057-a47f-4b03-9372-471e2723f03d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00137625, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.59395911404863, "model_name": "gemini-3.8-flash", "output_tokens": 144, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:53.359477+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00137625, "created_at": "2026-09-17T03:19:53.359Z" }, { "sequence": 2085, "attempt_id": "87ea8ea9-c94b-4fea-b967-da73c51b57ec", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011588, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 7.980633445084095, "model_name": "gemini-3.1-pro-preview", "output_tokens": 772, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:54.937205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011588, "created_at": "2026-09-17T03:19:54.937Z" }, { "sequence": 2086, "attempt_id": "026579a3-7add-476f-90ca-4f2f13109b5b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00440875, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.1163951030466706, "model_name": "muse-spark-1.2", "output_tokens": 300, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:55.476156+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00440875, "created_at": "2026-09-17T03:19:55.476Z" }, { "sequence": 2087, "attempt_id": "056ad784-8d9e-47d7-9ddf-183032094418", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00541, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.3664915029658005, "model_name": "gemini-3.1-pro-preview", "output_tokens": 265, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:55.901617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00541, "created_at": "2026-09-17T03:19:55.901Z" }, { "sequence": 2088, "attempt_id": "f974f2e8-91f6-4732-af7e-9932c1a71251", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.530803058994934, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:56.550111+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T03:19:56.550Z" }, { "sequence": 2089, "attempt_id": "a99522e1-a292-4340-9bc4-56e39e683845", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004702, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.6712237850297242, "model_name": "muse-spark-1.3", "output_tokens": 369, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:56.894814+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004702, "created_at": "2026-09-17T03:19:56.894Z" }, { "sequence": 2090, "attempt_id": "48e3e12b-836c-4bdc-8245-79786d23bac0", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.006714, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 9.743025718024, "model_name": "muse-spark-1.3", "output_tokens": 828, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:57.360745+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006714, "created_at": "2026-09-17T03:19:57.360Z" }, { "sequence": 2091, "attempt_id": "0d810f4a-2ed1-420c-ad63-3f2503868849", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0109665, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 10.485313226934522, "model_name": "gemini-3.8-flash", "output_tokens": 2692, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:57.609053+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0109665, "created_at": "2026-09-17T03:19:57.609Z" }, { "sequence": 2092, "attempt_id": "024aec65-084d-4b06-9632-96274683be80", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0061, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.1329939799616113, "model_name": "claude-sonnet-5", "output_tokens": 50, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:57.719702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0061, "created_at": "2026-09-17T03:19:57.719Z" }, { "sequence": 2093, "attempt_id": "2000d222-2bf8-46e7-9ecf-3ac2eefbc390", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.2036638610297814, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:58.205113+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:19:58.205Z" }, { "sequence": 2094, "attempt_id": "3020ff45-a856-4832-9a99-b9d52794b315", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 0.9418522369815037, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:58.661878+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:19:58.662Z" }, { "sequence": 2095, "attempt_id": "53cefd7c-1c34-4e9a-8120-b7239855ef45", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 5.465248306980357, "model_name": "claude-fable-5-1", "output_tokens": 15, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:58.929344+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:19:58.929Z" }, { "sequence": 2096, "attempt_id": "74c67b05-fd27-480a-9030-4b9e56d48d00", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005276, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.5926480330526829, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:59.061906+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005276, "created_at": "2026-09-17T03:19:59.062Z" }, { "sequence": 2097, "attempt_id": "511984b0-1c58-431a-8fd8-c3753a796c0c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 4.1829379689879715, "model_name": "claude-opus-5", "output_tokens": 51, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:59.227302+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015275, "created_at": "2026-09-17T03:19:59.227Z" }, { "sequence": 2098, "attempt_id": "bfadfc82-17f3-47f6-93ba-deb77208dae8", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.664266107021831, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:59.333426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:19:59.333Z" }, { "sequence": 2099, "attempt_id": "918e6e27-d1d8-4247-84af-4c6a85e7ca65", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8905846739653498, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:59.661029+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:19:59.661Z" }, { "sequence": 2100, "attempt_id": "7bd0f244-d7e4-4d71-a512-ecfa1c4a047e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0024015, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.9631278020096943, "model_name": "gemini-3.8-flash", "output_tokens": 415, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:00.275386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0024015, "created_at": "2026-09-17T03:20:00.275Z" }, { "sequence": 2101, "attempt_id": "3e9cc795-cb9a-4f50-909d-ffafe981bceb", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.5262800980126485, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:00.909745+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:20:00.909Z" }, { "sequence": 2102, "attempt_id": "e70e7e95-5400-40eb-b815-95f3bd112275", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01298725, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 27.388376454007812, "model_name": "muse-spark-1.3", "output_tokens": 2312, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:01.018153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01298725, "created_at": "2026-09-17T03:20:01.018Z" }, { "sequence": 2103, "attempt_id": "11eaf48d-613f-4659-91fb-c8fd03239a62", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.1188946830807254, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:01.556022+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01447, "created_at": "2026-09-17T03:20:01.556Z" }, { "sequence": 2104, "attempt_id": "87ec7479-8f79-4132-ae73-78d4deef586e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006418, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.1001968829659745, "model_name": "claude-sonnet-5", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:01.866298+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006418, "created_at": "2026-09-17T03:20:01.866Z" }, { "sequence": 2105, "attempt_id": "dfed54cd-0899-40f7-ad88-ed1c3aa1d4c4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006586, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.257238844991662, "model_name": "gemini-3.1-pro-preview", "output_tokens": 361, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:02.084160+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006586, "created_at": "2026-09-17T03:20:02.084Z" }, { "sequence": 2106, "attempt_id": "00718834-abdc-43f7-835a-ba6c5155b921", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02356975, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 28.28650942898821, "model_name": "muse-spark-1.2", "output_tokens": 4802, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:02.201953+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02356975, "created_at": "2026-09-17T03:20:02.202Z" }, { "sequence": 2107, "attempt_id": "4a92d8a0-07af-4258-8458-8c760b29baf8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.015469, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 14.688429589034058, "model_name": "muse-spark-1.2", "output_tokens": 2888, "prediction": { "choice": "B" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:02.408579+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015469, "created_at": "2026-09-17T03:20:02.408Z" }, { "sequence": 2108, "attempt_id": "ca3598d2-f4e8-4efb-9665-f3a26c40649d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.403323512058705, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:02.737044+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:20:02.737Z" }, { "sequence": 2109, "attempt_id": "36fbf864-0cf6-4475-a48f-7013e8f6d1e3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6176558940205723, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:02.846356+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:20:02.846Z" }, { "sequence": 2110, "attempt_id": "3cc2be84-dedb-4110-9816-66569329fe46", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7405636650510132, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:02.952226+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:20:02.952Z" }, { "sequence": 2111, "attempt_id": "b3f8da01-f9c2-4915-8f8d-ad61ba84bf17", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.9770120660541579, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:03.061685+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004968, "created_at": "2026-09-17T03:20:03.061Z" }, { "sequence": 2112, "attempt_id": "b8e4d304-293b-4475-b8b6-0d40f7ef218c", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.05435878993012, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:03.571675+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:20:03.571Z" }, { "sequence": 2113, "attempt_id": "57266cad-d994-46c9-b381-8659750438a8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.3671110660070553, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:04.032900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:20:04.033Z" }, { "sequence": 2114, "attempt_id": "fcfbb634-37d5-4bb4-9754-add9201a48ea", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005133, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 5.425197658012621, "model_name": "muse-spark-1.3", "output_tokens": 466, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:04.460793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005133, "created_at": "2026-09-17T03:20:04.460Z" }, { "sequence": 2115, "attempt_id": "71bb0400-5bac-4dc4-b399-17a6f3c3671f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00256875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.286740691983141, "model_name": "gemini-3.8-flash", "output_tokens": 455, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:04.592808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00256875, "created_at": "2026-09-17T03:20:04.592Z" }, { "sequence": 2116, "attempt_id": "0ace3fcc-3129-467b-81ce-f13e57870fdb", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 1.1537323299562559, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:05.295056+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T03:20:05.295Z" }, { "sequence": 2117, "attempt_id": "b4902fe4-ea14-4165-9e6b-877fd8632b57", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.8790506310760975, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:05.558129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T03:20:05.558Z" }, { "sequence": 2118, "attempt_id": "eb759f29-4227-40fb-82ce-19c54272cc96", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1345466529019177, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:05.831691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:20:05.831Z" }, { "sequence": 2119, "attempt_id": "f837bd9f-672d-423b-adcc-f24114472a8a", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.9075438419822603, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:06.079756+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014315, "created_at": "2026-09-17T03:20:06.079Z" }, { "sequence": 2120, "attempt_id": "b5e42a50-2354-4054-993e-db196c167107", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.274813943076879, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:06.336773+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T03:20:06.336Z" }, { "sequence": 2121, "attempt_id": "65fb9601-4b61-485b-ac66-1953037d6efb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00819725, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 7.200914154062048, "model_name": "muse-spark-1.2", "output_tokens": 1187, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:06.444896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00819725, "created_at": "2026-09-17T03:20:06.445Z" }, { "sequence": 2122, "attempt_id": "6c42b161-ce90-4a4c-bf00-66c04b13974d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.8777866089949384, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:06.550317+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:20:06.550Z" }, { "sequence": 2123, "attempt_id": "64984d5f-c364-4b3c-b6fd-584549f93f1b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0460967799881473, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:06.651350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:20:06.651Z" }, { "sequence": 2124, "attempt_id": "5314eaec-65ab-4677-9bab-93820f4c852e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.4050529019441456, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:06.985144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:20:06.985Z" }, { "sequence": 2125, "attempt_id": "e0aff845-e3ba-43a4-adb4-747a77b88ab6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0062065, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.265611186972819, "model_name": "muse-spark-1.2", "output_tokens": 713, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:07.324353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0062065, "created_at": "2026-09-17T03:20:07.324Z" }, { "sequence": 2126, "attempt_id": "28361790-a880-4e74-be0c-96602dc04018", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.9165299449814484, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:07.431956+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:20:07.433Z" }, { "sequence": 2127, "attempt_id": "e4400fac-5060-4cb4-a008-a3c17a594f2a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.5399056329624727, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:08.079101+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001833, "created_at": "2026-09-17T03:20:08.079Z" }, { "sequence": 2128, "attempt_id": "6ab5045a-e007-42ff-8121-8d898c6ffe81", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0017715, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.9874720129882917, "model_name": "gemini-3.8-flash", "output_tokens": 245, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:08.189850+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0017715, "created_at": "2026-09-17T03:20:08.191Z" }, { "sequence": 2129, "attempt_id": "0d63e639-6e01-46ce-abf7-97aa25f6ac86", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009944, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 7.052551407017745, "model_name": "gemini-3.1-pro-preview", "output_tokens": 637, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:09.254819+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009944, "created_at": "2026-09-17T03:20:09.254Z" }, { "sequence": 2130, "attempt_id": "942a712f-2f79-41c3-b85d-50ca325503a1", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.8466897730249912, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:09.364909+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005684, "created_at": "2026-09-17T03:20:09.365Z" }, { "sequence": 2131, "attempt_id": "3d832d6c-c1b1-4817-a8ff-8ab6123791da", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004596, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.7126339180395007, "model_name": "muse-spark-1.2", "output_tokens": 337, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:09.471354+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004596, "created_at": "2026-09-17T03:20:09.471Z" }, { "sequence": 2132, "attempt_id": "d0d59c09-e1ab-487f-b556-92547c3b44e1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.8121916110394523, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:10.283843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004904, "created_at": "2026-09-17T03:20:10.284Z" }, { "sequence": 2133, "attempt_id": "aec3a74a-8595-41f4-b842-fa3eb575b8a4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.1529681030660868, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:10.394942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02438, "created_at": "2026-09-17T03:20:10.395Z" }, { "sequence": 2134, "attempt_id": "d36da5d6-357f-433c-bd40-84b036f702ce", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.9607562039745972, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:10.508030+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004904, "created_at": "2026-09-17T03:20:10.508Z" }, { "sequence": 2135, "attempt_id": "b16b5bca-4a1b-4d69-82f1-d44d915fd8d3", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00645, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.716169400024228, "model_name": "gemini-3.1-pro-preview", "output_tokens": 348, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:10.656761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00645, "created_at": "2026-09-17T03:20:10.656Z" }, { "sequence": 2136, "attempt_id": "0c27fae2-8aa7-49fe-99db-d25e69e54685", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 3.699882964952849, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:10.802784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01421, "created_at": "2026-09-17T03:20:10.802Z" }, { "sequence": 2137, "attempt_id": "fd6128c3-90b1-4d79-9f24-277518a9b315", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.6740092179970816, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:10.969194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009752, "created_at": "2026-09-17T03:20:10.969Z" }, { "sequence": 2138, "attempt_id": "2e378413-aaa4-4568-bf4c-fd9fcd75be94", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00510175, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 4.533221489051357, "model_name": "muse-spark-1.3", "output_tokens": 456, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:11.083784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00510175, "created_at": "2026-09-17T03:20:11.083Z" }, { "sequence": 2139, "attempt_id": "7a750809-70d3-493f-85db-5ff79f024057", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 4.364495599991642, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:11.188417+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02844, "created_at": "2026-09-17T03:20:11.190Z" }, { "sequence": 2140, "attempt_id": "8bbd8b04-88cb-4943-8845-56730934f37a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066995, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 8.476745955063961, "model_name": "muse-spark-1.3", "output_tokens": 829, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:11.323363+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0066995, "created_at": "2026-09-17T03:20:11.323Z" }, { "sequence": 2141, "attempt_id": "02d95d1f-eacb-425f-8236-e8cebc179733", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.8855716140242293, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:11.498421+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:20:11.498Z" }, { "sequence": 2142, "attempt_id": "7b5594dd-3e75-42e2-81e0-5e1051d88c8d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001155, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.2642717739799991, "model_name": "gemini-3.8-flash", "output_tokens": 85, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:11.659471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001155, "created_at": "2026-09-17T03:20:11.659Z" }, { "sequence": 2143, "attempt_id": "1677a9c7-bb7a-471a-9ca2-93efff68601b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00372275, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.002737027942203, "model_name": "muse-spark-1.2", "output_tokens": 138, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:11.778202+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00372275, "created_at": "2026-09-17T03:20:11.778Z" }, { "sequence": 2144, "attempt_id": "c1483f37-27a6-468c-af87-76f01083a541", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001879, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.7708842800930142, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:12.065413+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001879, "created_at": "2026-09-17T03:20:12.065Z" }, { "sequence": 2145, "attempt_id": "1317664f-8978-4543-86db-7d07fae5d18a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005704, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6409634209703654, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:13.419485+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005704, "created_at": "2026-09-17T03:20:13.419Z" }, { "sequence": 2146, "attempt_id": "ae52ef00-c64b-4d7c-acc1-9a7fa10bac3a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.9232757861027494, "model_name": "gpt-5.6-sol", "output_tokens": 70, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:13.530803+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011056, "created_at": "2026-09-17T03:20:13.531Z" }, { "sequence": 2147, "attempt_id": "c96183e6-5dd5-449b-9c68-8f55f2480f4c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005974, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.150075888959691, "model_name": "gemini-3.1-pro-preview", "output_tokens": 312, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:13.727232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005974, "created_at": "2026-09-17T03:20:13.727Z" }, { "sequence": 2148, "attempt_id": "0bd06958-ddd7-43dc-b58b-b7504ce3d27b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00456, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 3.319592054001987, "model_name": "muse-spark-1.3", "output_tokens": 335, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:13.935308+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00456, "created_at": "2026-09-17T03:20:13.935Z" }, { "sequence": 2149, "attempt_id": "8a1e5f75-0d0f-48e8-80d4-3b144fcb62a3", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036359999999999995, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.8521636930527166, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:14.489677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036359999999999995, "created_at": "2026-09-17T03:20:14.489Z" }, { "sequence": 2150, "attempt_id": "2b63b413-db24-45eb-8759-ec388cc13823", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 3.199804740957916, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:14.627732+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02484, "created_at": "2026-09-17T03:20:14.627Z" }, { "sequence": 2151, "attempt_id": "256e0d46-a89d-4af9-b16a-30bd7784dc25", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000568, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.8027990139089525, "model_name": "gpt-5.6-luna", "output_tokens": 71, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:14.734562+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000568, "created_at": "2026-09-17T03:20:14.734Z" }, { "sequence": 2152, "attempt_id": "c95a7065-1556-4e35-9153-c069daaf747c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01854, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 4.1164323459379375, "model_name": "claude-opus-5", "output_tokens": 168, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:15.191169+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01854, "created_at": "2026-09-17T03:20:15.191Z" }, { "sequence": 2153, "attempt_id": "029b65de-2efe-476a-924a-2ed40b23b7cb", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00287025, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 2.59082025103271, "model_name": "gemini-3.8-flash", "output_tokens": 533, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:16.121928+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00287025, "created_at": "2026-09-17T03:20:16.122Z" }, { "sequence": 2154, "attempt_id": "d9b1c127-d3d5-4c5f-bc2c-f3958c810564", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.195412194938399, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:16.508993+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:20:16.509Z" }, { "sequence": 2155, "attempt_id": "59d0205c-712c-47d8-b052-92a4ecb1e9fb", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02975, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 5.701614364050329, "model_name": "claude-fable-5-1", "output_tokens": 21, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:16.617611+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02975, "created_at": "2026-09-17T03:20:16.617Z" }, { "sequence": 2156, "attempt_id": "e3dbe42b-ee2e-4c05-88b3-f36485e6bdeb", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.863409185083583, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:16.725062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:20:16.725Z" }, { "sequence": 2157, "attempt_id": "45a38df1-e0d0-4344-abf9-817087d966ff", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.4164606620324776, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:17.151290+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:20:17.151Z" }, { "sequence": 2158, "attempt_id": "1cdb2f54-857f-4611-9cff-c6efb69c70e5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.7466178860049695, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:17.580544+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:20:17.580Z" }, { "sequence": 2159, "attempt_id": "77dcf7e5-c150-467a-9bb3-de438dacb8cb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1168239449616522, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:17.842151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:20:17.842Z" }, { "sequence": 2160, "attempt_id": "a9ff3987-3fb3-4ad2-b295-31cf8dc630a6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.3287584519712254, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:17.951941+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:20:17.952Z" }, { "sequence": 2161, "attempt_id": "184243e6-9878-4d97-bb89-2d6055668941", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.1061713750241324, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:18.340367+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:20:18.340Z" }, { "sequence": 2162, "attempt_id": "dc376bd2-186d-4a26-979d-6b0b86b72fca", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 4.046527134021744, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:18.651464+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:20:18.651Z" }, { "sequence": 2163, "attempt_id": "ad373621-bde2-45da-850b-50b1b3cf8bc7", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.1162010510452092, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:19.068426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:20:19.068Z" }, { "sequence": 2164, "attempt_id": "8ff68733-cab9-4761-b503-432a17ed8a90", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.3991187430219725, "model_name": "gemini-3.8-flash", "output_tokens": 107, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:19.179097+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00125175, "created_at": "2026-09-17T03:20:19.179Z" }, { "sequence": 2165, "attempt_id": "df5b0fd9-0d14-41c3-b318-a0667556e8e5", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011486, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 8.0484708850272, "model_name": "claude-sonnet-5", "output_tokens": 575, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:19.287734+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011486, "created_at": "2026-09-17T03:20:19.287Z" }, { "sequence": 2166, "attempt_id": "91933fd7-6e44-4082-a495-720bcabb34ba", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00408225, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6552431270247325, "model_name": "muse-spark-1.3", "output_tokens": 217, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:19.714884+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00408225, "created_at": "2026-09-17T03:20:19.715Z" }, { "sequence": 2167, "attempt_id": "2fcd8fe2-0deb-45bc-bac6-bdf40f35cada", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01268, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 7.660069461911917, "model_name": "gemini-3.1-pro-preview", "output_tokens": 863, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:19.832457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01268, "created_at": "2026-09-17T03:20:19.832Z" }, { "sequence": 2168, "attempt_id": "8dad8a14-5a2e-4180-adfb-604a59097028", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004044, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6659641630249098, "model_name": "muse-spark-1.2", "output_tokens": 208, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:20.132113+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004044, "created_at": "2026-09-17T03:20:20.132Z" }, { "sequence": 2169, "attempt_id": "2340ef47-03fa-475b-aaee-9603649aa6ba", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00760225, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 6.321251746965572, "model_name": "muse-spark-1.2", "output_tokens": 1037, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:20.364969+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00760225, "created_at": "2026-09-17T03:20:20.365Z" }, { "sequence": 2170, "attempt_id": "2a559cec-44a0-44e8-a9cf-933cf9865ef5", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00504, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.2081567409913987, "model_name": "gemini-3.1-pro-preview", "output_tokens": 231, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:20.473535+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00504, "created_at": "2026-09-17T03:20:20.474Z" }, { "sequence": 2171, "attempt_id": "3370a7e5-5925-423b-ac2c-471037ed4282", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001925, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 1.1935201600426808, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:20.594144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001925, "created_at": "2026-09-17T03:20:20.594Z" }, { "sequence": 2172, "attempt_id": "f802c179-c14e-4039-81a9-c1e341dbec04", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00580875, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 6.863545258063823, "model_name": "muse-spark-1.3", "output_tokens": 615, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:20.704359+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00580875, "created_at": "2026-09-17T03:20:20.705Z" }, { "sequence": 2173, "attempt_id": "2c73262a-e016-440c-900d-fceff28ab497", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.108828139025718, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:21.048936+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:20:21.049Z" }, { "sequence": 2174, "attempt_id": "ea81a3bd-d121-4fd5-955f-5da8b14554ce", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.564132860978134, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:21.388297+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:20:21.388Z" }, { "sequence": 2175, "attempt_id": "b0bd8b33-68ca-4fea-8c5c-81685c78432f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.0562504030531272, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:21.870556+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:20:21.870Z" }, { "sequence": 2176, "attempt_id": "0c1b76ca-d14f-40c2-afd3-7d7e68800a96", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005742000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.4351794109679759, "model_name": "gpt-5.6-luna", "output_tokens": 68, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:21.982099+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005742000000000001, "created_at": "2026-09-17T03:20:21.983Z" }, { "sequence": 2177, "attempt_id": "b2657495-59d5-454c-9589-89a5b8d86a64", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005406, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.7369510040152818, "model_name": "gpt-5.6-terra", "output_tokens": 40, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:22.088263+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005406, "created_at": "2026-09-17T03:20:22.088Z" }, { "sequence": 2178, "attempt_id": "1a3ea76e-0a9d-49e6-9f91-051e07a24e9b", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00663, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.727523957961239, "model_name": "claude-sonnet-5", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:22.197482+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00663, "created_at": "2026-09-17T03:20:22.197Z" }, { "sequence": 2179, "attempt_id": "2dfaed44-5825-4d41-a3ad-e869c22e7557", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01765, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.0384609249886125, "model_name": "claude-opus-5", "output_tokens": 116, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:22.307692+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01765, "created_at": "2026-09-17T03:20:22.309Z" }, { "sequence": 2180, "attempt_id": "194e3f37-d0b0-49c3-99ab-8945e67dd954", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.5349948490038514, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:22.411768+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:20:22.411Z" }, { "sequence": 2181, "attempt_id": "2a71f1f5-118e-4c89-b244-97abef78a480", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": true, "cost_usd": 0.020819999999999998, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 2.4746212100144476, "model_name": "gemini-3.8-flash", "output_tokens": 214, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:23.179320+00:00", "request_attempts": 2, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 1, "valid": true, "within_bands": null }, "cost_usd": 0.020819999999999998, "created_at": "2026-09-17T03:20:23.179Z" }, { "sequence": 2182, "attempt_id": "bae2ee9c-9bb4-4052-b987-59b009771376", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 1.1825741119682789, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:23.490549+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001856, "created_at": "2026-09-17T03:20:23.490Z" }, { "sequence": 2183, "attempt_id": "87646e74-3af3-4c0c-9050-96e1203f7bfa", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00502, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.7796575740212575, "model_name": "muse-spark-1.2", "output_tokens": 415, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:24.279823+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00502, "created_at": "2026-09-17T03:20:24.280Z" }, { "sequence": 2184, "attempt_id": "42f3b987-a662-40e1-86ca-79201ce78f28", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.9510197109775618, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:24.480808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02454, "created_at": "2026-09-17T03:20:24.480Z" }, { "sequence": 2185, "attempt_id": "f6e8d39f-c55b-4b1d-9a8a-6bccb4d25550", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010696, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.0537717069964856, "model_name": "gpt-5.6-sol", "output_tokens": 58, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:24.592015+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010696, "created_at": "2026-09-17T03:20:24.592Z" }, { "sequence": 2186, "attempt_id": "1e9fc0af-2220-4e2a-9803-50cfd21892f8", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006662, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.438442953978665, "model_name": "claude-sonnet-5", "output_tokens": 101, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:24.703179+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006662, "created_at": "2026-09-17T03:20:24.704Z" }, { "sequence": 2187, "attempt_id": "0115f6ea-029c-4aba-aa06-c344ba4aba2e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005476, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.3030235539190471, "model_name": "gpt-5.6-terra", "output_tokens": 59, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:24.808359+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005476, "created_at": "2026-09-17T03:20:24.808Z" }, { "sequence": 2188, "attempt_id": "67a582bc-b23a-414f-a892-1196660e9c11", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006116, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.520851625013165, "model_name": "gemini-3.1-pro-preview", "output_tokens": 308, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:25.115271+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006116, "created_at": "2026-09-17T03:20:25.115Z" }, { "sequence": 2189, "attempt_id": "2a45e232-a74f-4e16-b0c6-ad0cfb4a0384", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003543, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.8720822840696201, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:25.575542+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003543, "created_at": "2026-09-17T03:20:25.575Z" }, { "sequence": 2190, "attempt_id": "faa206ff-7d87-4f53-a142-ec90a416e313", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016255, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.5452271009562537, "model_name": "claude-opus-5", "output_tokens": 85, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:25.739860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016255, "created_at": "2026-09-17T03:20:25.740Z" }, { "sequence": 2191, "attempt_id": "04da0e49-83de-48b8-b2c3-7770e9fb4a0a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005727999999999999, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.3913976489566267, "model_name": "gpt-5.6-luna", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:26.994514+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005727999999999999, "created_at": "2026-09-17T03:20:26.994Z" }, { "sequence": 2192, "attempt_id": "40e26731-ab32-401e-87d4-d958a67c4785", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02903, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.1393680709879845, "model_name": "claude-fable-5-1", "output_tokens": 15, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:27.121715+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02903, "created_at": "2026-09-17T03:20:27.121Z" }, { "sequence": 2193, "attempt_id": "74a60c46-4a7a-40b4-bd7d-6ca425b2f97c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.8184166129212826, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:27.668709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:20:27.668Z" }, { "sequence": 2194, "attempt_id": "7e8f5959-ad3d-4f7a-a49c-9f215dbf53dd", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.696687203948386, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:27.803917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:20:27.804Z" }, { "sequence": 2195, "attempt_id": "616cbe8d-86ee-4bf5-9716-fd278ab508d9", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.268117563915439, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:27.966044+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:20:27.966Z" }, { "sequence": 2196, "attempt_id": "6d5eec0b-513c-4a5d-a3c7-204bcbf1b5ef", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006023, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 7.074936916003935, "model_name": "muse-spark-1.3", "output_tokens": 651, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:28.234998+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006023, "created_at": "2026-09-17T03:20:28.235Z" }, { "sequence": 2197, "attempt_id": "4b041b18-7947-499d-8bc4-ff47367c6928", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9133556860033423, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:28.695172+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:20:28.695Z" }, { "sequence": 2198, "attempt_id": "c92d183e-60a3-40ec-863f-474188551cf9", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.4880438189720735, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:28.808161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:20:28.808Z" }, { "sequence": 2199, "attempt_id": "e2f3dd49-1d28-46ea-94fd-340fa3111157", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6969293979927897, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:28.930863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:20:28.931Z" }, { "sequence": 2200, "attempt_id": "2ca5ffbf-d091-45eb-8339-724d2df77e3b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005068, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.4410297069698572, "model_name": "gpt-5.6-terra", "output_tokens": 24, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:29.353867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005068, "created_at": "2026-09-17T03:20:29.354Z" }, { "sequence": 2201, "attempt_id": "af64a66e-7a81-4e16-a342-742bb8e8aed8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2992979809641838, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:29.466657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:20:29.466Z" }, { "sequence": 2202, "attempt_id": "62b314e5-d5ee-4fb3-bddd-d8a2c05cb9ad", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.9777489609550685, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:29.908900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:20:29.909Z" }, { "sequence": 2203, "attempt_id": "8cdd4e7f-adab-49a7-986f-3f319f95695d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0013065, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.2896674459334463, "model_name": "gemini-3.8-flash", "output_tokens": 121, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:30.098107+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0013065, "created_at": "2026-09-17T03:20:30.098Z" }, { "sequence": 2204, "attempt_id": "3d59feb2-3ee1-45d3-a2ff-915a6983e741", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0062895, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 5.564051093067974, "model_name": "gemini-3.8-flash", "output_tokens": 1451, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:30.208489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0062895, "created_at": "2026-09-17T03:20:30.208Z" }, { "sequence": 2205, "attempt_id": "43a8a1c8-59d9-443f-8a75-4fa619b3b187", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00400525, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.132495478959754, "model_name": "muse-spark-1.2", "output_tokens": 198, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:30.599457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00400525, "created_at": "2026-09-17T03:20:30.599Z" }, { "sequence": 2206, "attempt_id": "2cbe8dd5-d814-4148-8e5f-67130e8c4288", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-002", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.7932344099972397, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:31.111793+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T03:20:31.111Z" }, { "sequence": 2207, "attempt_id": "a21c92cf-82a5-45c7-a61e-eb1751449f1a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004239, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.375518771004863, "model_name": "muse-spark-1.3", "output_tokens": 253, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:31.416713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004239, "created_at": "2026-09-17T03:20:31.416Z" }, { "sequence": 2208, "attempt_id": "c7e337e5-e9df-48c6-a289-fac0f0381d26", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004134, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.386068621999584, "model_name": "gemini-3.1-pro-preview", "output_tokens": 155, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:31.729582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004134, "created_at": "2026-09-17T03:20:31.729Z" }, { "sequence": 2209, "attempt_id": "47355630-6a9d-4ab1-9de7-7d7e5f66a67f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005738, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 1.7593899499624968, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:31.968160+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005738, "created_at": "2026-09-17T03:20:31.968Z" }, { "sequence": 2210, "attempt_id": "d6e4aa2b-17d5-4fa0-b778-092535e7728a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.7522210900206119, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:32.467621+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:20:32.467Z" }, { "sequence": 2211, "attempt_id": "3466ebd0-7085-4b9a-837f-50999c2788c5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014345, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.4777735819807276, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:32.579347+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014345, "created_at": "2026-09-17T03:20:32.579Z" }, { "sequence": 2212, "attempt_id": "6280b043-dc3c-40ab-9994-73ec42ae7236", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.8604478050256148, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:32.703284+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:20:32.703Z" }, { "sequence": 2213, "attempt_id": "804b1994-061e-4fd8-8ff1-13936d19f222", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006387, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 7.837557581020519, "model_name": "muse-spark-1.3", "output_tokens": 759, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:32.813173+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006387, "created_at": "2026-09-17T03:20:32.813Z" }, { "sequence": 2214, "attempt_id": "c46245fe-dc50-444f-8e4f-716c8962c734", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.615574129973538, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:33.192974+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T03:20:33.193Z" }, { "sequence": 2215, "attempt_id": "089dc026-3ba9-4c64-8ed4-3b5c6768fd9c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.9771269970806316, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:33.304550+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:20:33.304Z" }, { "sequence": 2216, "attempt_id": "50aa70cc-0a42-4ad8-8392-c4ffb1b22ab5", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035580000000000003, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.9037415160564706, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:33.595354+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00035580000000000003, "created_at": "2026-09-17T03:20:33.595Z" }, { "sequence": 2217, "attempt_id": "14b155e1-4d11-4fc8-9a11-600fd10bfa37", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00175575, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.5835727190133184, "model_name": "gemini-3.8-flash", "output_tokens": 241, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:34.163211+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00175575, "created_at": "2026-09-17T03:20:34.163Z" }, { "sequence": 2218, "attempt_id": "8fa9a314-d0f4-417a-adb2-607ebef86c6a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00527, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.6944514939095825, "model_name": "gpt-5.6-terra", "output_tokens": 41, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:34.277034+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00527, "created_at": "2026-09-17T03:20:34.277Z" }, { "sequence": 2219, "attempt_id": "2760ee71-03de-4d54-9bd0-2b1a172dfc73", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010977, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 9.662924465024844, "model_name": "muse-spark-1.2", "output_tokens": 1839, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:34.582068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010977, "created_at": "2026-09-17T03:20:34.582Z" }, { "sequence": 2220, "attempt_id": "c72b554e-a554-4066-812b-b5ef09ea9be6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.0967257610755041, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:35.374061+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:20:35.374Z" }, { "sequence": 2221, "attempt_id": "3da7c842-e3ac-4e3b-918b-e4cadec3a29d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.079825217020698, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:35.504926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:20:35.505Z" }, { "sequence": 2222, "attempt_id": "ce757ecc-91f7-4795-b8e7-2c8869764cc2", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7976116710342467, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:35.614775+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:20:35.616Z" }, { "sequence": 2223, "attempt_id": "3c03328c-c5cf-4d73-8159-f82a402a817c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571675, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.098198502091691, "model_name": "muse-spark-1.2", "output_tokens": 601, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:36.021503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00571675, "created_at": "2026-09-17T03:20:36.021Z" }, { "sequence": 2224, "attempt_id": "ddd37702-3320-40ca-a8a0-c7b9bd7332cb", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7806406879099086, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:36.166663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:20:36.166Z" }, { "sequence": 2225, "attempt_id": "693af285-89e2-42b5-84b5-562addb6a8d5", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006292, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.239256981993094, "model_name": "gemini-3.1-pro-preview", "output_tokens": 335, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:36.316975+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006292, "created_at": "2026-09-17T03:20:36.317Z" }, { "sequence": 2226, "attempt_id": "cbb05513-ac92-47d6-8bf5-afb75e918ca0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0905593580100685, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:36.577946+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:20:36.578Z" }, { "sequence": 2227, "attempt_id": "d037c66b-a79d-4e9a-81ff-9ed49621aa6b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004939, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.810643221018836, "model_name": "muse-spark-1.3", "output_tokens": 418, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:36.692422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004939, "created_at": "2026-09-17T03:20:36.692Z" }, { "sequence": 2228, "attempt_id": "65956d63-c24b-422e-8224-cfb699226a29", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.3443655730225146, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:37.039585+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:20:37.039Z" }, { "sequence": 2229, "attempt_id": "19c8eef4-d28d-43c5-aab4-348b77382444", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003552, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.0564438719302416, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:37.332765+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003552, "created_at": "2026-09-17T03:20:37.332Z" }, { "sequence": 2230, "attempt_id": "a6c2dc41-2b4b-47a7-8186-b023531064e9", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00128175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.206258254009299, "model_name": "gemini-3.8-flash", "output_tokens": 115, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:37.446316+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00128175, "created_at": "2026-09-17T03:20:37.446Z" }, { "sequence": 2231, "attempt_id": "3217b313-3294-48a8-bdc2-e0b4c47bbf65", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9905252519529313, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:37.605558+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:20:37.605Z" }, { "sequence": 2232, "attempt_id": "781d1052-b016-4d26-8c58-fd416c5cff12", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00401, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.4457778369542211, "model_name": "muse-spark-1.2", "output_tokens": 200, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:38.138486+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00401, "created_at": "2026-09-17T03:20:38.138Z" }, { "sequence": 2233, "attempt_id": "5a22e007-794e-478b-bf8a-f637fbdc6ff7", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.732733684941195, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:38.291300+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T03:20:38.291Z" }, { "sequence": 2234, "attempt_id": "e9623883-a8a5-49da-8de2-102be080a64d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.020094, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 14.018715235055424, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1486, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:38.413337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.020094, "created_at": "2026-09-17T03:20:38.413Z" }, { "sequence": 2235, "attempt_id": "99a748b6-57e8-46fa-9623-5eb7eb968e36", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004265, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.1232223669067025, "model_name": "muse-spark-1.3", "output_tokens": 260, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:38.549781+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004265, "created_at": "2026-09-17T03:20:38.549Z" }, { "sequence": 2236, "attempt_id": "7a55ca53-3caa-4eaa-bfc6-244d6e87b2c1", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005052, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.4719762828899547, "model_name": "gemini-3.1-pro-preview", "output_tokens": 232, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:39.197426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005052, "created_at": "2026-09-17T03:20:39.197Z" }, { "sequence": 2237, "attempt_id": "9c6b9b12-9b29-473c-a668-c8d7597b7bc8", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.5807208640035242, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:39.314000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:20:39.314Z" }, { "sequence": 2238, "attempt_id": "d98497a3-8c11-47aa-a7cd-19436175d0ed", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 0.7854763229843229, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:39.422421+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:20:39.422Z" }, { "sequence": 2239, "attempt_id": "1b14461d-478f-40f7-b221-31d33bb04fb0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.0064119179733098, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:39.528786+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:20:39.528Z" }, { "sequence": 2240, "attempt_id": "68b58928-ad7f-4811-994a-ed1f8bd261d2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005494, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.4194080930901691, "model_name": "gpt-5.6-terra", "output_tokens": 59, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:39.820096+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005494, "created_at": "2026-09-17T03:20:39.820Z" }, { "sequence": 2241, "attempt_id": "f532e358-be3c-4127-89d3-9915cc8b14da", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.2357888789847493, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:40.038049+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T03:20:40.038Z" }, { "sequence": 2242, "attempt_id": "33974c74-252d-4708-9ce9-5acd4dc71fdd", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015415, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 3.2606541459681466, "model_name": "claude-opus-5", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:40.413269+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015415, "created_at": "2026-09-17T03:20:40.413Z" }, { "sequence": 2243, "attempt_id": "d0695a92-f95b-4cf2-ae43-991e465022cf", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.9971439059590921, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:40.635774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:20:40.635Z" }, { "sequence": 2244, "attempt_id": "3a9532a3-6a67-4e41-b05c-6f4bfc6c0dc0", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006816, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 3.207823913078755, "model_name": "claude-sonnet-5", "output_tokens": 119, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:40.749337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006816, "created_at": "2026-09-17T03:20:40.750Z" }, { "sequence": 2245, "attempt_id": "2bd7adc7-2bc7-4600-a1ea-64dd9ef5de0e", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.7567058720160276, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:41.506348+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T03:20:41.506Z" }, { "sequence": 2246, "attempt_id": "15856f39-d72d-49b7-bf8a-17b23d9ddf49", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004704, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.807069333968684, "model_name": "gemini-3.1-pro-preview", "output_tokens": 202, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:42.466382+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004704, "created_at": "2026-09-17T03:20:42.466Z" }, { "sequence": 2247, "attempt_id": "3e1cc8ed-eb6e-4990-b222-4b2ee5cf3872", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00611, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.9961060180794448, "model_name": "claude-sonnet-5", "output_tokens": 51, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:42.587685+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00611, "created_at": "2026-09-17T03:20:42.588Z" }, { "sequence": 2248, "attempt_id": "8cdb709a-4d1c-4bdb-a895-2103076432f2", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00157875, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.17343122290913, "model_name": "gemini-3.8-flash", "output_tokens": 193, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:42.697933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00157875, "created_at": "2026-09-17T03:20:42.698Z" }, { "sequence": 2249, "attempt_id": "172b64b5-7667-43cc-bd37-203e291a0c57", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.0251875510439277, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:42.823964+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:20:42.824Z" }, { "sequence": 2250, "attempt_id": "2a565be8-b936-4d73-97e9-5aad32aaf06e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 9.548461873084307, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:42.937791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:20:42.938Z" }, { "sequence": 2251, "attempt_id": "00af77d5-ed51-42cd-a289-d86831f84c38", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00469025, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.4732385240495205, "model_name": "muse-spark-1.3", "output_tokens": 358, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:43.114938+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00469025, "created_at": "2026-09-17T03:20:43.115Z" }, { "sequence": 2252, "attempt_id": "d64ec08d-fe77-48ff-bd6d-5aa0991a1696", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.4346824520034716, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:43.378158+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02857, "created_at": "2026-09-17T03:20:43.378Z" }, { "sequence": 2253, "attempt_id": "02d83469-7759-4f77-9216-cfa5f63792f3", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.8044826090335846, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:43.669969+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:20:43.670Z" }, { "sequence": 2254, "attempt_id": "8845db55-3801-4423-a02a-66073c552877", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 0.9577505890047178, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:43.784346+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:20:43.784Z" }, { "sequence": 2255, "attempt_id": "2ef54c41-10d4-490f-9c9c-ab21f9d39bef", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00620325, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.161859157960862, "model_name": "muse-spark-1.2", "output_tokens": 714, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:43.898200+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00620325, "created_at": "2026-09-17T03:20:43.898Z" }, { "sequence": 2256, "attempt_id": "23a05100-3b29-4d79-8100-7dc97e6c3705", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.1625477219931781, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:44.211724+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:20:44.211Z" }, { "sequence": 2257, "attempt_id": "6455978c-f566-43ae-929f-6f7885423fde", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005324, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.6420651939697564, "model_name": "gpt-5.6-terra", "output_tokens": 47, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:44.337367+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005324, "created_at": "2026-09-17T03:20:44.338Z" }, { "sequence": 2258, "attempt_id": "15e2565c-cebe-4856-bafd-2749b14c23a2", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.9734950149431825, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:45.311160+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:20:45.311Z" }, { "sequence": 2259, "attempt_id": "30403d02-6376-4c89-a150-4d8bdafe7379", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6765949019463733, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:45.686238+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:20:45.686Z" }, { "sequence": 2260, "attempt_id": "13ce1ca5-1f0d-43e7-b595-9cbcd6af130d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00289275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 2.7956757760839537, "model_name": "gemini-3.8-flash", "output_tokens": 546, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:45.802460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00289275, "created_at": "2026-09-17T03:20:45.802Z" }, { "sequence": 2261, "attempt_id": "8d784d54-0840-41b3-b632-53a67500c25d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.0585083820624277, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:45.956973+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:20:45.957Z" }, { "sequence": 2262, "attempt_id": "0533b225-b57b-4bdc-806e-4f1263f318cc", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.954364636912942, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:46.403620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:20:46.403Z" }, { "sequence": 2263, "attempt_id": "cd026a89-f9bd-42e4-9a81-089f5119a017", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0059158210642636, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:46.519409+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:20:46.519Z" }, { "sequence": 2264, "attempt_id": "a0308314-6ae6-49aa-a53c-c923c8ab35ff", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0204, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 6.393805840052664, "model_name": "claude-opus-5", "output_tokens": 256, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:46.631916+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0204, "created_at": "2026-09-17T03:20:46.632Z" }, { "sequence": 2265, "attempt_id": "a58ac3c0-1b66-4cef-a278-5d1b66f36993", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0052, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2980983939487487, "model_name": "gpt-5.6-terra", "output_tokens": 35, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:47.100848+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0052, "created_at": "2026-09-17T03:20:47.100Z" }, { "sequence": 2266, "attempt_id": "70995042-ef88-4760-8744-e58af5b64360", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.643324153032154, "model_name": "gemini-3.1-pro-preview", "output_tokens": 395, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:47.467557+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006994, "created_at": "2026-09-17T03:20:47.467Z" }, { "sequence": 2267, "attempt_id": "3c7a2397-af09-48ff-ae16-577f99a34f44", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8590206179069355, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:47.583354+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003561, "created_at": "2026-09-17T03:20:47.584Z" }, { "sequence": 2268, "attempt_id": "4c83f2d0-c052-44c2-aa64-311630deb294", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.303683356032707, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:48.088330+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:20:48.088Z" }, { "sequence": 2269, "attempt_id": "ff37fc28-ab40-41ea-ac5f-f4be1cf43af1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.475010753958486, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:48.392341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:20:48.392Z" }, { "sequence": 2270, "attempt_id": "bce4f1a2-687e-456c-9122-e8196e575af7", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00119775, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.259893708047457, "model_name": "gemini-3.8-flash", "output_tokens": 92, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:48.779579+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00119775, "created_at": "2026-09-17T03:20:48.779Z" }, { "sequence": 2271, "attempt_id": "0c90db1d-62d4-4d66-a806-dd93434ec7d1", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003162, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.2554974919185042, "model_name": "gemini-3.1-pro-preview", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:49.323787+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003162, "created_at": "2026-09-17T03:20:49.323Z" }, { "sequence": 2272, "attempt_id": "cbb47be8-6a67-4d2b-a232-059f9d441a8f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00739825, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 6.231243125046603, "model_name": "muse-spark-1.2", "output_tokens": 999, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:49.721016+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00739825, "created_at": "2026-09-17T03:20:49.721Z" }, { "sequence": 2273, "attempt_id": "2839e519-37b3-4d5a-8129-9336cb5aa553", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.2632231330499053, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:49.839562+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:20:49.839Z" }, { "sequence": 2274, "attempt_id": "38faf285-4f9a-48a7-9d21-c1866a35cc27", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.0282433460233733, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:50.234373+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:20:50.234Z" }, { "sequence": 2275, "attempt_id": "e2e730ad-c3f6-472d-9418-6c377c0b462e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.820346469990909, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:50.519034+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:20:50.519Z" }, { "sequence": 2276, "attempt_id": "9f7ad5d1-1782-417a-8676-78368bb56f88", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005463, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.4108391830231994, "model_name": "muse-spark-1.2", "output_tokens": 541, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:50.638524+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005463, "created_at": "2026-09-17T03:20:50.638Z" }, { "sequence": 2277, "attempt_id": "0fadc84c-c71b-4599-9527-64124f127782", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.8736191070638597, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:50.841375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:20:50.841Z" }, { "sequence": 2278, "attempt_id": "24f0b53a-e96d-4d5e-80a4-216c0eb931ef", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.9620658629573882, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:50.960324+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:20:50.961Z" }, { "sequence": 2279, "attempt_id": "18f451cf-f8d0-4b6c-ab3c-4134aafceedb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.630507018067874, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:51.077910+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:20:51.078Z" }, { "sequence": 2280, "attempt_id": "516cfa03-b361-4d38-a7df-4c258d8e4653", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00538375, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 7.852506910916418, "model_name": "muse-spark-1.3", "output_tokens": 525, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:51.200040+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00538375, "created_at": "2026-09-17T03:20:51.200Z" }, { "sequence": 2281, "attempt_id": "a07c699f-aa14-4b86-9679-78c9a923fc61", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.562754544080235, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:51.402584+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:20:51.402Z" }, { "sequence": 2282, "attempt_id": "e504199d-518a-44fb-a7a9-cb59d2c3a4c2", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.764431606978178, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:51.521474+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:20:51.521Z" }, { "sequence": 2283, "attempt_id": "b39fa5db-5074-4e2c-9c12-2cc7a2d7679b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.8657017110381275, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:51.634852+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:20:51.634Z" }, { "sequence": 2284, "attempt_id": "056f8177-c26d-45ed-809e-810e1d313a5f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00137025, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.1880276219453663, "model_name": "gemini-3.8-flash", "output_tokens": 142, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:51.822711+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00137025, "created_at": "2026-09-17T03:20:51.822Z" }, { "sequence": 2285, "attempt_id": "a880f58c-c4f0-424b-9d91-c5e7770599e7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00486375, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 5.5223972470266744, "model_name": "muse-spark-1.3", "output_tokens": 400, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:52.266581+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00486375, "created_at": "2026-09-17T03:20:52.266Z" }, { "sequence": 2286, "attempt_id": "b8c2e4d3-1059-4d70-afba-a8c7af98158e", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.6635722590144724, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:52.413282+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:20:52.413Z" }, { "sequence": 2287, "attempt_id": "454c77c3-8c34-4c6d-94b3-b8d9df5971da", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00443075, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.5305191460065544, "model_name": "muse-spark-1.2", "output_tokens": 304, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:52.608771+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00443075, "created_at": "2026-09-17T03:20:52.608Z" }, { "sequence": 2288, "attempt_id": "816040fc-7b27-403c-9199-1cc0f78cee52", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1180187891004607, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:53.056520+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:20:53.056Z" }, { "sequence": 2289, "attempt_id": "7202e0b8-c313-490d-a65f-dfbcc22a1a68", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.7050390880322084, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:53.226779+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T03:20:53.226Z" }, { "sequence": 2290, "attempt_id": "876558d3-380d-42f3-a0a5-450d5d73ec83", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004226, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.950854792026803, "model_name": "gemini-3.1-pro-preview", "output_tokens": 166, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:53.354885+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004226, "created_at": "2026-09-17T03:20:53.355Z" }, { "sequence": 2291, "attempt_id": "ba95a5af-6e10-493c-97c2-2e803538ec1b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.813024349976331, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:20:53.566454+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:20:53.566Z" }, { "sequence": 2292, "attempt_id": "d08a9881-1f9c-4301-99f4-54bb18edc55d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00438825, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.653142602997832, "model_name": "muse-spark-1.3", "output_tokens": 294, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:53.683388+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00438825, "created_at": "2026-09-17T03:20:53.683Z" }, { "sequence": 2293, "attempt_id": "e4d9b079-d0dc-43b9-956c-bb2e97bfdec2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00534, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4324233559891582, "model_name": "gpt-5.6-terra", "output_tokens": 42, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:53.817653+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00534, "created_at": "2026-09-17T03:20:53.817Z" }, { "sequence": 2294, "attempt_id": "c1b11618-aabc-4f85-bd51-e412cca3470e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.8954552749637514, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:54.250617+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003764, "created_at": "2026-09-17T03:20:54.250Z" }, { "sequence": 2295, "attempt_id": "ffefa8ea-41f3-464e-9bbb-9533194c1803", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016775, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.1403631230350584, "model_name": "claude-opus-5", "output_tokens": 101, "prediction": { "choice": "D" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:20:54.469992+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016775, "created_at": "2026-09-17T03:20:54.470Z" }, { "sequence": 2296, "attempt_id": "9d60b237-2f29-40e8-ad6e-e4c8a3e23bac", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.5085732920560986, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:20:54.708879+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:20:54.709Z" }, { "sequence": 2297, "attempt_id": "3d5c837f-6595-4e38-996a-7d9dc3e4ae27", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.503313688095659, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:55.089760+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001884, "created_at": "2026-09-17T03:20:55.089Z" }, { "sequence": 2298, "attempt_id": "84782144-c081-4cc6-a516-75cebd0bf5b0", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00340725, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.7845325460657477, "model_name": "gemini-3.8-flash", "output_tokens": 676, "prediction": { "choice": "D" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:20:55.961050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00340725, "created_at": "2026-09-17T03:20:55.961Z" }, { "sequence": 2299, "attempt_id": "b935a783-26c9-4cab-8202-1f1f8b03c012", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.664481128915213, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:56.490842+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02484, "created_at": "2026-09-17T03:20:56.491Z" }, { "sequence": 2300, "attempt_id": "60510e1a-c8a5-4fae-9d65-58d0856fcee4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0006288, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.966019746963866, "model_name": "gpt-5.6-luna", "output_tokens": 121, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:56.612555+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006288, "created_at": "2026-09-17T03:20:56.612Z" }, { "sequence": 2301, "attempt_id": "8e0c5230-cdd7-4e07-a782-4738c3fef22e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007046, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.0402870930265635, "model_name": "claude-sonnet-5", "output_tokens": 131, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:57.409889+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007046, "created_at": "2026-09-17T03:20:57.410Z" }, { "sequence": 2302, "attempt_id": "27401c90-8030-413f-a425-bac75b4c05b6", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018165, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.7310939090093598, "model_name": "claude-opus-5", "output_tokens": 153, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:57.665922+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.018165, "created_at": "2026-09-17T03:20:57.666Z" }, { "sequence": 2303, "attempt_id": "59185cf1-83f1-4667-b532-92343253906b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005692000000000001, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.5806838279822841, "model_name": "gpt-5.6-luna", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:58.193539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005692000000000001, "created_at": "2026-09-17T03:20:58.193Z" }, { "sequence": 2304, "attempt_id": "05ffa803-262d-49b1-b0bf-803cdd46e545", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005692, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.1393535879906267, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:58.315981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005692, "created_at": "2026-09-17T03:20:58.316Z" }, { "sequence": 2305, "attempt_id": "1fa1145c-ba5a-49f3-92d4-cba8390e986a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02945, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 4.546628422103822, "model_name": "claude-fable-5-1", "output_tokens": 15, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:58.457274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02945, "created_at": "2026-09-17T03:20:58.459Z" }, { "sequence": 2306, "attempt_id": "deede765-5f8b-41ce-a047-0e8d056526df", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011736, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 3.1249799290671945, "model_name": "gpt-5.6-sol", "output_tokens": 104, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:58.575311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011736, "created_at": "2026-09-17T03:20:58.575Z" }, { "sequence": 2307, "attempt_id": "2d7c3409-d323-455f-998b-baf147042630", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.007572, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 5.0052459600847214, "model_name": "muse-spark-1.2", "output_tokens": 1029, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:58.697342+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007572, "created_at": "2026-09-17T03:20:58.697Z" }, { "sequence": 2308, "attempt_id": "b220b220-97a7-45d1-b81c-05b1b4b80aac", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.8366459099343047, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:58.812478+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003761, "created_at": "2026-09-17T03:20:58.812Z" }, { "sequence": 2309, "attempt_id": "9be4fa5b-943e-4f88-9987-21341ce87fe1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-020", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.8097340340027586, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:59.746713+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:20:59.746Z" }, { "sequence": 2310, "attempt_id": "1061cdf2-ad1a-44ff-8531-9aa6816ddbc0", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01183, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 7.967105546966195, "model_name": "gemini-3.1-pro-preview", "output_tokens": 792, "prediction": { "choice": "D" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:21:00.691404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01183, "created_at": "2026-09-17T03:21:00.691Z" }, { "sequence": 2311, "attempt_id": "c360a608-b539-4b23-8a00-2ddeeef8cf49", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002634, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 4.149328193976544, "model_name": "gemini-3.8-flash", "output_tokens": 470, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:21:01.681731+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002634, "created_at": "2026-09-17T03:21:01.681Z" }, { "sequence": 2312, "attempt_id": "64f05a52-b07b-4616-9700-40ade47a9abb", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02636, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.132234551012516, "model_name": "gpt-6-astra", "output_tokens": 34, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:02.000958+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02636, "created_at": "2026-09-17T03:21:02.001Z" }, { "sequence": 2313, "attempt_id": "f3a8b09d-4ac2-4ff9-9cbe-ec9fb5b524ac", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008218, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.5538153080269694, "model_name": "claude-sonnet-5", "output_tokens": 235, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:02.487399+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008218, "created_at": "2026-09-17T03:21:02.487Z" }, { "sequence": 2314, "attempt_id": "ef5ae975-e187-4a5c-9cfe-9b40df9223fd", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03016, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.9578621711116284, "model_name": "claude-fable-5-1", "output_tokens": 16, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:21:02.652018+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03016, "created_at": "2026-09-17T03:21:02.652Z" }, { "sequence": 2315, "attempt_id": "8c54eb51-358f-48ac-a9be-49a7670256a8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009464, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 6.065125687047839, "model_name": "gemini-3.1-pro-preview", "output_tokens": 595, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:02.794904+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009464, "created_at": "2026-09-17T03:21:02.795Z" }, { "sequence": 2316, "attempt_id": "11d2b09d-d811-474f-9ede-e4a3feeb4503", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011284, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.9874565299833193, "model_name": "gpt-5.6-sol", "output_tokens": 71, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:02.913080+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011284, "created_at": "2026-09-17T03:21:02.913Z" }, { "sequence": 2317, "attempt_id": "cd93ea74-24c8-4512-bda1-3881867cd145", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8366094869561493, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "google", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:21:03.749989+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003789, "created_at": "2026-09-17T03:21:03.750Z" }, { "sequence": 2318, "attempt_id": "0a3940f3-9215-4cd0-9c36-558426452349", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005832, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.9657263139961287, "model_name": "gpt-5.6-terra", "output_tokens": 75, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:03.873155+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005832, "created_at": "2026-09-17T03:21:03.873Z" }, { "sequence": 2319, "attempt_id": "9d8fcc77-8cad-4bba-9074-c906dbc24bdb", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02277, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 6.483687781030312, "model_name": "claude-opus-5", "output_tokens": 324, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:05.181332+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02277, "created_at": "2026-09-17T03:21:05.181Z" }, { "sequence": 2320, "attempt_id": "b09eb60f-f831-489e-bc72-055458bc1918", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0028935, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.4326633700402454, "model_name": "gemini-3.8-flash", "output_tokens": 529, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:21:05.304564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0028935, "created_at": "2026-09-17T03:21:05.305Z" }, { "sequence": 2321, "attempt_id": "1dabd492-c478-437e-8f99-bce0334b2c24", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005964000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.3407087059458718, "model_name": "gpt-5.6-luna", "output_tokens": 86, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:05.461018+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005964000000000001, "created_at": "2026-09-17T03:21:05.461Z" }, { "sequence": 2322, "attempt_id": "937ef1dc-a373-4254-9d77-bcb45356c460", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.5105005570221692, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:06.099855+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T03:21:06.100Z" }, { "sequence": 2323, "attempt_id": "7b648551-72b6-46a9-bcb3-5d6f8abe26cb", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01432, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 1.9085243979934603, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:07.213409+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01432, "created_at": "2026-09-17T03:21:07.213Z" }, { "sequence": 2324, "attempt_id": "d5f81629-ba10-4e57-b237-e4e55dd864af", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0093575, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 8.79694194206968, "model_name": "muse-spark-1.2", "output_tokens": 1450, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:07.339367+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0093575, "created_at": "2026-09-17T03:21:07.340Z" }, { "sequence": 2325, "attempt_id": "b8aa5654-c651-41e4-9b10-c8de1b968d00", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.276497737970203, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:07.453977+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T03:21:07.454Z" }, { "sequence": 2326, "attempt_id": "95d8fa49-047b-4cd2-8d86-10d393e1603e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.6576178609393537, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:07.880862+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:21:07.881Z" }, { "sequence": 2327, "attempt_id": "60ad7b77-8f58-47f3-8284-250cb65c5010", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006518, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.8477774289203808, "model_name": "claude-sonnet-5", "output_tokens": 90, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:08.272382+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006518, "created_at": "2026-09-17T03:21:08.272Z" }, { "sequence": 2328, "attempt_id": "086a5830-a161-4fa2-b567-4829b85ef014", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010772, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.6022517569363117, "model_name": "gpt-5.6-sol", "output_tokens": 61, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:08.941933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010772, "created_at": "2026-09-17T03:21:08.942Z" }, { "sequence": 2329, "attempt_id": "147ee05a-d426-4b8e-886f-3cce1774c480", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005484, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.7099022230831906, "model_name": "gpt-5.6-terra", "output_tokens": 59, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:09.280143+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005484, "created_at": "2026-09-17T03:21:09.280Z" }, { "sequence": 2330, "attempt_id": "65f99ba8-5f1d-44c1-a8b5-5eec10169a13", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.6965326819336042, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:09.760823+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003555, "created_at": "2026-09-17T03:21:09.760Z" }, { "sequence": 2331, "attempt_id": "26571765-53b5-4754-aa7d-29b2a709ee7b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01055975, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 16.30803009599913, "model_name": "muse-spark-1.3", "output_tokens": 1732, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:09.888271+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01055975, "created_at": "2026-09-17T03:21:09.889Z" }, { "sequence": 2332, "attempt_id": "d64ca8a1-8368-4383-9a95-dc5e2089919c", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00205875, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.8293864410370588, "model_name": "gemini-3.8-flash", "output_tokens": 322, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:21:10.220767+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00205875, "created_at": "2026-09-17T03:21:10.220Z" }, { "sequence": 2333, "attempt_id": "ac07e74a-5c5b-4415-9815-da9ddab789d4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.6963416149374098, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:10.340375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004944, "created_at": "2026-09-17T03:21:10.340Z" }, { "sequence": 2334, "attempt_id": "78a55406-1370-4267-8d68-1be76014dba4", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.428717163973488, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:11.887615+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00571, "created_at": "2026-09-17T03:21:11.887Z" }, { "sequence": 2335, "attempt_id": "18a9857f-8ceb-4cd4-9be0-8e72bb2808ae", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014342, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 9.516599971917458, "model_name": "gemini-3.1-pro-preview", "output_tokens": 993, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "google", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:21:12.135840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014342, "created_at": "2026-09-17T03:21:12.135Z" }, { "sequence": 2336, "attempt_id": "b9e49825-d6ad-4ca5-9cef-5936ef13db2e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0080485, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 14.1977550330339, "model_name": "muse-spark-1.3", "output_tokens": 1142, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:12.514035+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0080485, "created_at": "2026-09-17T03:21:12.514Z" }, { "sequence": 2337, "attempt_id": "672e6819-87e1-46f9-beb9-38dd50fe3d0d", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 1.3455338600324467, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:13.356047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T03:21:13.356Z" }, { "sequence": 2338, "attempt_id": "daa21282-d531-4365-a8b3-d7b38f73bb65", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 3.043444160022773, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:13.495844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014275, "created_at": "2026-09-17T03:21:13.496Z" }, { "sequence": 2339, "attempt_id": "438756f0-c7da-4033-9286-e398b184ce27", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0058005, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 3.6018577340291813, "model_name": "muse-spark-1.2", "output_tokens": 621, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:13.611377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0058005, "created_at": "2026-09-17T03:21:13.611Z" }, { "sequence": 2340, "attempt_id": "2454bddc-6822-4c8e-a3e5-fa7ab081fde0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.0835365910315886, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:13.734925+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:21:13.735Z" }, { "sequence": 2341, "attempt_id": "2c4b022b-3ea5-49d8-861e-691c1591e6b9", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008174, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.5469445899361745, "model_name": "gemini-3.1-pro-preview", "output_tokens": 492, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:13.853544+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008174, "created_at": "2026-09-17T03:21:13.853Z" }, { "sequence": 2342, "attempt_id": "6cda24e1-f237-4c8b-a2bf-d5b0f0c75340", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.76550324796699, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:13.975498+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02857, "created_at": "2026-09-17T03:21:13.976Z" }, { "sequence": 2343, "attempt_id": "534eb7ce-f523-42e3-95bf-00ec9323916c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.6246223950292915, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:14.089856+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:21:14.089Z" }, { "sequence": 2344, "attempt_id": "228e0333-71f1-4b21-a58a-0b095c433215", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00508225, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 4.341189622064121, "model_name": "muse-spark-1.3", "output_tokens": 452, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:14.204517+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00508225, "created_at": "2026-09-17T03:21:14.204Z" }, { "sequence": 2345, "attempt_id": "c7cf5574-cb33-4562-93e4-1850a83d168b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 0.7461297509726137, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:14.477942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:21:14.478Z" }, { "sequence": 2346, "attempt_id": "ee7ec416-7b7e-4c19-9838-20e903b8e4dd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0053, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.3813212929526344, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:14.877489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0053, "created_at": "2026-09-17T03:21:14.877Z" }, { "sequence": 2347, "attempt_id": "9265ef9f-7168-43ee-9df7-eadf8f8a543a", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.9949227250181139, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:15.024122+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:21:15.024Z" }, { "sequence": 2348, "attempt_id": "befdff7d-9f44-44f3-9516-0fb5e9ea67b0", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00164775, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.4833136819070205, "model_name": "gemini-3.8-flash", "output_tokens": 214, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:15.455656+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00164775, "created_at": "2026-09-17T03:21:15.455Z" }, { "sequence": 2349, "attempt_id": "d8ebfcbd-45c2-464c-990f-e2f798e505d3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00380275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.4954862029990181, "model_name": "muse-spark-1.2", "output_tokens": 153, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:15.820737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00380275, "created_at": "2026-09-17T03:21:15.820Z" }, { "sequence": 2350, "attempt_id": "f0ec20a3-87c8-45db-900e-3e5cd3668943", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.875006131012924, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:16.020556+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T03:21:16.020Z" }, { "sequence": 2351, "attempt_id": "67c8ea38-ba9f-45da-a080-6c8f4f5d7839", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006018, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.020336564979516, "model_name": "claude-sonnet-5", "output_tokens": 40, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:17.044786+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006018, "created_at": "2026-09-17T03:21:17.044Z" }, { "sequence": 2352, "attempt_id": "354d47e7-d49a-4b38-8eca-0562e09c3896", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00463, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 3.8003926440142095, "model_name": "gemini-3.1-pro-preview", "output_tokens": 198, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:17.535590+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00463, "created_at": "2026-09-17T03:21:17.535Z" }, { "sequence": 2353, "attempt_id": "b3a0ab60-062f-401a-9323-24baa61c3dbc", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.0636834800243378, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:17.657442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:21:17.658Z" }, { "sequence": 2354, "attempt_id": "07d08d30-e3ea-46e4-ac66-65f5179e9f1f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005388, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.6457828019047156, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:17.787475+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005388, "created_at": "2026-09-17T03:21:17.787Z" }, { "sequence": 2355, "attempt_id": "fba53810-7a75-4f37-88d0-5fadccc96184", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018745, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 3.719460439053364, "model_name": "claude-opus-5", "output_tokens": 188, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:18.343867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.018745, "created_at": "2026-09-17T03:21:18.344Z" }, { "sequence": 2356, "attempt_id": "abce13e1-3ef4-49d8-b68d-c3b976fcea7c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 4.054514944902621, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:18.469118+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T03:21:18.469Z" }, { "sequence": 2357, "attempt_id": "385ba468-2c10-43e6-8e60-8e5a7fc42178", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011652, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.521467344951816, "model_name": "gpt-5.6-sol", "output_tokens": 105, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:18.584713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011652, "created_at": "2026-09-17T03:21:18.584Z" }, { "sequence": 2358, "attempt_id": "fc79bd8f-6bf4-4a4e-88ca-d73fbf80453c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00466125, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 4.3244752519531175, "model_name": "muse-spark-1.3", "output_tokens": 355, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:18.705875+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00466125, "created_at": "2026-09-17T03:21:18.706Z" }, { "sequence": 2359, "attempt_id": "4c67cd5a-be9c-4bc9-b2f8-44e3ff7724f1", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.031755618052557, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:18.950528+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003555, "created_at": "2026-09-17T03:21:18.950Z" }, { "sequence": 2360, "attempt_id": "7b64b817-f895-40cf-9a65-fa70a04f4126", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.6026723940158263, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:19.771183+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004944, "created_at": "2026-09-17T03:21:19.771Z" }, { "sequence": 2361, "attempt_id": "5e43f666-aace-47cf-9ab1-e718d553d723", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00942125, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 16.86148356902413, "model_name": "muse-spark-1.3", "output_tokens": 1450, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:19.896700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00942125, "created_at": "2026-09-17T03:21:19.896Z" }, { "sequence": 2362, "attempt_id": "27461f44-e26a-44a7-a1a9-804d3256fb6a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.7122178559657186, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:20.609225+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001876, "created_at": "2026-09-17T03:21:20.609Z" }, { "sequence": 2363, "attempt_id": "009ef5dd-53f3-4b6f-82ed-f4b091ef8721", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01399425, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 16.890427712001838, "model_name": "muse-spark-1.2", "output_tokens": 2526, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:21:20.763861+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01399425, "created_at": "2026-09-17T03:21:20.763Z" }, { "sequence": 2364, "attempt_id": "9918650d-ba20-4bd3-a30a-367dddf72093", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.077913255081512, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:21.151905+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005798, "created_at": "2026-09-17T03:21:21.152Z" }, { "sequence": 2365, "attempt_id": "0a5b91e3-2513-45e3-8aaa-e38517678206", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015795, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.651968408958055, "model_name": "claude-opus-5", "output_tokens": 63, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:21.476821+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015795, "created_at": "2026-09-17T03:21:21.476Z" }, { "sequence": 2366, "attempt_id": "e1b64e2f-be13-47a4-bb4b-c7270c0dfd1b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.7770795350661501, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:21.806703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02474, "created_at": "2026-09-17T03:21:21.806Z" }, { "sequence": 2367, "attempt_id": "70fbe473-b84b-409f-af90-2899a5d10273", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0032325, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 4.397105638985522, "model_name": "gemini-3.8-flash", "output_tokens": 635, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:22.174507+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0032325, "created_at": "2026-09-17T03:21:22.174Z" }, { "sequence": 2368, "attempt_id": "e60afca9-dc04-404b-a1c9-c04da2b5ab7b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010796, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.908884795033373, "model_name": "gpt-5.6-sol", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:22.641475+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010796, "created_at": "2026-09-17T03:21:22.641Z" }, { "sequence": 2369, "attempt_id": "8fb2ff0d-faf9-4eb6-8810-bc429976db3e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 3.9401558820391074, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:22.766208+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02901, "created_at": "2026-09-17T03:21:22.767Z" }, { "sequence": 2370, "attempt_id": "022d0ca3-d698-4a41-818d-8047411e30c6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005624, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.9072029379894957, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:22.900304+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005624, "created_at": "2026-09-17T03:21:22.900Z" }, { "sequence": 2371, "attempt_id": "457becc7-6baa-4a06-a1ff-df44f74ddbbf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.6028654519468546, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:23.018434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004976000000000001, "created_at": "2026-09-17T03:21:23.018Z" }, { "sequence": 2372, "attempt_id": "905911b0-d0b9-4f1c-a6ea-a93ace0c4ef4", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.083451158949174, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:23.380630+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000374, "created_at": "2026-09-17T03:21:23.380Z" }, { "sequence": 2373, "attempt_id": "be030cdb-0445-4cbf-aac6-0468a5306db6", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009662, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 6.150250214035623, "model_name": "gemini-3.1-pro-preview", "output_tokens": 616, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:23.807963+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009662, "created_at": "2026-09-17T03:21:23.808Z" }, { "sequence": 2374, "attempt_id": "f187586e-2061-4b2e-aca4-286de6d744aa", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 1.132455668062903, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:25.065011+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T03:21:25.065Z" }, { "sequence": 2375, "attempt_id": "df4e5ad7-f3f3-4891-98f1-a44e201af39f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00195375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 3.2095064390450716, "model_name": "gemini-3.8-flash", "output_tokens": 290, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:25.191497+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00195375, "created_at": "2026-09-17T03:21:25.193Z" }, { "sequence": 2376, "attempt_id": "7c70f2c7-77f5-446e-aeaa-2658855730aa", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.162842526915483, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:25.313050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T03:21:25.313Z" }, { "sequence": 2377, "attempt_id": "9117cb85-22a6-4619-89c1-7f37f30d9c06", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.8992383549921215, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:25.436106+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T03:21:25.436Z" }, { "sequence": 2378, "attempt_id": "ff9ef0cd-bb37-422f-8c7a-601168648529", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 0.8947736809495836, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:26.331246+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:21:26.331Z" }, { "sequence": 2379, "attempt_id": "eaddce9b-3229-4a24-8f9b-10abd1582095", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1350263169733807, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:26.460516+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:21:26.460Z" }, { "sequence": 2380, "attempt_id": "cf3f064f-0e22-4119-9a18-6182dff66e81", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0047665, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 3.8126083010574803, "model_name": "muse-spark-1.3", "output_tokens": 373, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:26.584386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0047665, "created_at": "2026-09-17T03:21:26.584Z" }, { "sequence": 2381, "attempt_id": "44ff949a-6a5d-4d82-9ee1-503797cb37ac", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.509569860994816, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:26.705794+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T03:21:26.707Z" }, { "sequence": 2382, "attempt_id": "aed461ec-e650-4e64-8af4-61b3e42cb331", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.8937858159188181, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:27.085568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:21:27.085Z" }, { "sequence": 2383, "attempt_id": "beebae40-7951-453f-a974-02680f07b117", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009006, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 5.756578528904356, "model_name": "gemini-3.1-pro-preview", "output_tokens": 558, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:27.357258+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009006, "created_at": "2026-09-17T03:21:27.357Z" }, { "sequence": 2384, "attempt_id": "61cc5e3e-e453-424c-b791-a80538207fec", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0476396949961782, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:27.753725+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:21:27.753Z" }, { "sequence": 2385, "attempt_id": "8b98d97d-006a-4c99-b764-ce4323a67929", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0013575, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.6076133579481393, "model_name": "gemini-3.8-flash", "output_tokens": 139, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:28.192301+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0013575, "created_at": "2026-09-17T03:21:28.192Z" }, { "sequence": 2386, "attempt_id": "975392d3-487c-49ae-8d91-ec20c0d0b52c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01071775, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 10.164172252058052, "model_name": "muse-spark-1.2", "output_tokens": 1778, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:28.866906+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01071775, "created_at": "2026-09-17T03:21:28.867Z" }, { "sequence": 2387, "attempt_id": "195e13de-58a0-4f84-8f59-54ab80cf42dd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 3.4891060950467363, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:29.058298+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:21:29.058Z" }, { "sequence": 2388, "attempt_id": "03b9962f-443b-4322-98ea-cd62b9c077f9", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00349, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 3.068550181109458, "model_name": "gemini-3.1-pro-preview", "output_tokens": 105, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:29.529386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00349, "created_at": "2026-09-17T03:21:29.529Z" }, { "sequence": 2389, "attempt_id": "1e9c18b9-f10c-47e5-a723-29c5fe05b39a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.6795538100413978, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:29.674278+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:21:29.674Z" }, { "sequence": 2390, "attempt_id": "52767f9f-e33e-4b78-986c-30fdd5193021", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.3773766789818183, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:29.794275+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:21:29.794Z" }, { "sequence": 2391, "attempt_id": "1de18dfb-b79a-4648-a36d-789684e9795a", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.189311734982766, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:30.079529+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:21:30.079Z" }, { "sequence": 2392, "attempt_id": "d2d56e6a-070e-484a-be6d-ea8cc6f40d5a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051695, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.2064100599382073, "model_name": "muse-spark-1.2", "output_tokens": 479, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:30.415949+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0051695, "created_at": "2026-09-17T03:21:30.416Z" }, { "sequence": 2393, "attempt_id": "8cc6ab3a-a38d-4ae7-837d-8348c6c1f14e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2471777059836313, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:30.900887+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:21:30.901Z" }, { "sequence": 2394, "attempt_id": "1b94f579-83e0-4ba8-a578-1b07f19c84f1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1806626769248396, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:31.027923+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:21:31.028Z" }, { "sequence": 2395, "attempt_id": "de065392-6257-4fab-8892-0b0463b3be88", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7256433219881728, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:31.146435+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:21:31.146Z" }, { "sequence": 2396, "attempt_id": "b65c47cb-36fc-4c43-ab4d-83b86c67c553", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009977, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 8.842077783076093, "model_name": "muse-spark-1.2", "output_tokens": 1599, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:31.742680+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009977, "created_at": "2026-09-17T03:21:31.742Z" }, { "sequence": 2397, "attempt_id": "c6f27732-fb18-40f2-9783-409f28e31deb", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001239, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.225702491006814, "model_name": "gemini-3.8-flash", "output_tokens": 107, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:31.869020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001239, "created_at": "2026-09-17T03:21:31.870Z" }, { "sequence": 2398, "attempt_id": "37a05cd8-4fc2-42e1-ae9b-24a2d1d1efee", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7804537810152397, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:21:31.988935+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:21:31.989Z" }, { "sequence": 2399, "attempt_id": "de3cd02c-b9d1-4780-938e-e28bd0cb7171", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.507002404076047, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:32.112471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:21:32.112Z" }, { "sequence": 2400, "attempt_id": "620e8b11-17be-4c2e-953b-387462a13019", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00563275, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 5.527881628018804, "model_name": "muse-spark-1.3", "output_tokens": 588, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:32.354769+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00563275, "created_at": "2026-09-17T03:21:32.354Z" }, { "sequence": 2401, "attempt_id": "5d8e54ca-9c04-46ec-afa7-c1d648707e34", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.502249787095934, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:32.477438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:21:32.477Z" }, { "sequence": 2402, "attempt_id": "2bd6e3ed-65e5-4b65-ae0e-01ba60a7f7aa", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00406525, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.6619438549969345, "model_name": "muse-spark-1.3", "output_tokens": 218, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:32.930141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00406525, "created_at": "2026-09-17T03:21:32.930Z" }, { "sequence": 2403, "attempt_id": "334d1859-4505-453c-8b45-a47a0b488b4e", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003518, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.002968106069602, "model_name": "gemini-3.1-pro-preview", "output_tokens": 107, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:33.206655+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003518, "created_at": "2026-09-17T03:21:33.206Z" }, { "sequence": 2404, "attempt_id": "dd5907a5-b441-48e7-b466-e42f56ce36ad", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.1988353540655226, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:33.445913+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:21:33.446Z" }, { "sequence": 2405, "attempt_id": "765429e5-d611-4e11-ad60-bf68be0167bd", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006218, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 1.4037302699871361, "model_name": "claude-sonnet-5", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:33.569262+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006218, "created_at": "2026-09-17T03:21:33.569Z" }, { "sequence": 2406, "attempt_id": "4cc419a4-d15a-4bb7-a1b9-e19ef5b58a15", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044265, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.227851976058446, "model_name": "muse-spark-1.2", "output_tokens": 303, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:33.687413+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0044265, "created_at": "2026-09-17T03:21:33.687Z" }, { "sequence": 2407, "attempt_id": "0c867b5f-1f56-45d2-b6b9-b090c05fa4fd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.1510669670533389, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:33.811372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:21:33.811Z" }, { "sequence": 2408, "attempt_id": "b2411a3e-39ac-44a9-8ea7-a2147e43d673", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.0206846160581335, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:34.129877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01447, "created_at": "2026-09-17T03:21:34.130Z" }, { "sequence": 2409, "attempt_id": "450ba488-4a25-44dd-b6fa-d4458f9a0150", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.72255479299929, "model_name": "gpt-5.6-terra", "output_tokens": 53, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:34.777692+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005436, "created_at": "2026-09-17T03:21:34.777Z" }, { "sequence": 2410, "attempt_id": "261f0187-a0e2-490c-9b3d-c4e268e8dd3e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.1084294210886583, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:34.920046+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:21:34.920Z" }, { "sequence": 2411, "attempt_id": "98637a88-d0e6-4158-aecd-a90314557e52", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000564, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7913932380033657, "model_name": "gpt-5.6-luna", "output_tokens": 70, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:35.125661+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000564, "created_at": "2026-09-17T03:21:35.125Z" }, { "sequence": 2412, "attempt_id": "5cfb82c8-dc5e-41bc-bdca-7bb40cace0b3", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.6506151809589937, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:35.248867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:21:35.249Z" }, { "sequence": 2413, "attempt_id": "cbef9252-69e3-41c5-997f-397d5f089461", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00171, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.5862076879711822, "model_name": "gemini-3.8-flash", "output_tokens": 226, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:35.394238+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00171, "created_at": "2026-09-17T03:21:35.394Z" }, { "sequence": 2414, "attempt_id": "af8910c8-8029-46fc-82cf-67b87ba2973b", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 1.1017295399215072, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:36.473105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:21:36.473Z" }, { "sequence": 2415, "attempt_id": "7010272e-39a9-48ff-a83e-e57a15933454", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 2.8891125390073285, "model_name": "muse-spark-1.2", "output_tokens": 545, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:37.143095+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0054925, "created_at": "2026-09-17T03:21:37.143Z" }, { "sequence": 2416, "attempt_id": "10119a25-7b5e-4675-8a72-7687fe9049fb", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9496594640659168, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:37.283887+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T03:21:37.284Z" }, { "sequence": 2417, "attempt_id": "151bc7bc-d50a-4770-9cf6-ec0efb4712ac", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.648776601999998, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:37.403833+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:21:37.403Z" }, { "sequence": 2418, "attempt_id": "a9fd616e-1789-4721-8a4e-0af3121ad34b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 5.295351333916187, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:37.522627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:21:37.522Z" }, { "sequence": 2419, "attempt_id": "69300f4f-32a3-46f4-944b-cfb1894c74e4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006176, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.9550175149925053, "model_name": "gemini-3.1-pro-preview", "output_tokens": 323, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:37.653406+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006176, "created_at": "2026-09-17T03:21:37.655Z" }, { "sequence": 2420, "attempt_id": "77ab4188-9216-48dc-8e0d-87362b52bf5a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1255538170225918, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:37.780512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:21:37.780Z" }, { "sequence": 2421, "attempt_id": "92c41bfc-ab86-4fd7-be90-e2a89974660a", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.2463738380465657, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:38.290253+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T03:21:38.290Z" }, { "sequence": 2422, "attempt_id": "8833de68-2660-4aea-96c0-eec81dd4f16a", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-015", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.9608224689727649, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:38.741652+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036389999999999995, "created_at": "2026-09-17T03:21:38.741Z" }, { "sequence": 2423, "attempt_id": "f0eb5cc5-f46f-43d9-99e7-320075e76eb5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.8002635290613398, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:39.445582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:21:39.445Z" }, { "sequence": 2424, "attempt_id": "971f13e2-d9df-479d-a46d-e7486cefe444", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006012, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.228157651028596, "model_name": "gpt-5.6-terra", "output_tokens": 98, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:39.574350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006012, "created_at": "2026-09-17T03:21:39.574Z" }, { "sequence": 2425, "attempt_id": "03778fdf-9ee7-4c87-8a35-b06b3d2ece84", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006768, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.17608530796133, "model_name": "claude-sonnet-5", "output_tokens": 109, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:41.877101+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006768, "created_at": "2026-09-17T03:21:41.877Z" }, { "sequence": 2426, "attempt_id": "a180da2e-e106-42d8-b901-fcebac845208", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004341, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.255485474015586, "model_name": "gemini-3.8-flash", "output_tokens": 925, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:21:42.006471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004341, "created_at": "2026-09-17T03:21:42.008Z" }, { "sequence": 2427, "attempt_id": "a6419779-4cb4-42a5-9448-2db831f98af6", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 7.096835281001404, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:42.126703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:21:42.126Z" }, { "sequence": 2428, "attempt_id": "e43a1b69-da14-4a3d-a5df-634237952c8d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01602, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.767312241019681, "model_name": "claude-opus-5", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:42.341942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01602, "created_at": "2026-09-17T03:21:42.342Z" }, { "sequence": 2429, "attempt_id": "66387cef-1453-4903-b254-b1fb734bf920", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6845902169588953, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:42.691350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:21:42.691Z" }, { "sequence": 2430, "attempt_id": "69d11c88-2c56-432a-a970-f30630b5ed65", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.8298568360041827, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:42.817953+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:21:42.818Z" }, { "sequence": 2431, "attempt_id": "ff23e76f-0b45-4fb4-bc0e-d3940e6fb107", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.0590899749659002, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:43.877387+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004968, "created_at": "2026-09-17T03:21:43.877Z" }, { "sequence": 2432, "attempt_id": "905162a3-9843-4386-a06c-6a555a7bf1ac", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6498645480023697, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:44.005997+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:21:44.006Z" }, { "sequence": 2433, "attempt_id": "2ca8820c-4548-4d77-936a-404da63b65f4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005496, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6598255980061367, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:44.130816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005496, "created_at": "2026-09-17T03:21:44.131Z" }, { "sequence": 2434, "attempt_id": "59d64f53-a7ab-4494-8e20-208064d17970", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0127195, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 25.65650514292065, "model_name": "muse-spark-1.3", "output_tokens": 2249, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:44.251743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0127195, "created_at": "2026-09-17T03:21:44.251Z" }, { "sequence": 2435, "attempt_id": "eda4311c-44ac-4d46-8084-c65e1cb4149b", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011518, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 7.086901940056123, "model_name": "gemini-3.1-pro-preview", "output_tokens": 766, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:44.736413+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011518, "created_at": "2026-09-17T03:21:44.736Z" }, { "sequence": 2436, "attempt_id": "fd787054-2143-4d32-b7bb-f18ea8217ac6", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9143505740212277, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:21:45.045473+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:21:45.045Z" }, { "sequence": 2437, "attempt_id": "d9a0d709-2886-48e0-a8be-06c5b1f49806", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00709475, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 11.219408186036162, "model_name": "muse-spark-1.3", "output_tokens": 922, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:45.171362+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00709475, "created_at": "2026-09-17T03:21:45.172Z" }, { "sequence": 2438, "attempt_id": "b159343f-003b-4e61-be32-c8cad99cc471", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.4679506139364094, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:45.719926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:21:45.720Z" }, { "sequence": 2439, "attempt_id": "e562c17d-98d2-4661-9945-30eb2e323c51", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.4395100580295548, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:46.747650+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:21:46.747Z" }, { "sequence": 2440, "attempt_id": "9ab4536f-6fe8-4cda-a28f-2f0be40413c2", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.1849263879703358, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:47.032432+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:21:47.032Z" }, { "sequence": 2441, "attempt_id": "ad07f7c8-c5e1-4428-a9dd-d341212e66f2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00505475, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 2.844955071923323, "model_name": "muse-spark-1.2", "output_tokens": 442, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:47.221940+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00505475, "created_at": "2026-09-17T03:21:47.222Z" }, { "sequence": 2442, "attempt_id": "c2c4cb59-dfb9-4652-943a-a9b14bf8bace", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005108, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.310344657977112, "model_name": "gemini-3.1-pro-preview", "output_tokens": 234, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:47.350172+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005108, "created_at": "2026-09-17T03:21:47.351Z" }, { "sequence": 2443, "attempt_id": "59f13c50-3d48-482d-91e7-dee00b0300a2", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.105960029992275, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:47.473345+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:21:47.473Z" }, { "sequence": 2444, "attempt_id": "f28be4d4-d1a2-4fa2-a40d-a7be8ec810c3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01109525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 9.661268474068493, "model_name": "muse-spark-1.2", "output_tokens": 1858, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:48.078895+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01109525, "created_at": "2026-09-17T03:21:48.079Z" }, { "sequence": 2445, "attempt_id": "d491b9e1-a70e-4557-a516-794873b318f9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0235399330267683, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:48.374020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:21:48.374Z" }, { "sequence": 2446, "attempt_id": "0e7fafda-d996-45d9-b4c4-976d20cc12cf", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00341625, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.435791819007136, "model_name": "gemini-3.8-flash", "output_tokens": 681, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:48.512876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00341625, "created_at": "2026-09-17T03:21:48.513Z" }, { "sequence": 2447, "attempt_id": "e8bd2b53-be35-4608-b594-2a54988f2e41", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7975643000099808, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:48.674311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:21:48.674Z" }, { "sequence": 2448, "attempt_id": "fdfe7e38-8490-4760-8d41-be57515a895b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.8117207209579647, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:48.803936+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:21:48.804Z" }, { "sequence": 2449, "attempt_id": "8a38b735-e714-4f9e-9584-ba18d64f2f6d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.8227858360623941, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:21:49.335999+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:21:49.336Z" }, { "sequence": 2450, "attempt_id": "67ff511b-4fcc-4a9f-b1c2-bc09c99d3414", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.331124359043315, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:49.491746+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:21:49.491Z" }, { "sequence": 2451, "attempt_id": "0042513e-d168-4bc7-820f-ad16cc2bbbfe", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001173, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.409261084976606, "model_name": "gemini-3.8-flash", "output_tokens": 86, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:49.620463+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001173, "created_at": "2026-09-17T03:21:49.622Z" }, { "sequence": 2452, "attempt_id": "287b2c55-cd2e-45a1-9ac9-c3594343fcca", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00402275, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.297762028989382, "model_name": "muse-spark-1.2", "output_tokens": 203, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:50.102024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00402275, "created_at": "2026-09-17T03:21:50.102Z" }, { "sequence": 2453, "attempt_id": "980b9a3c-ba88-4c45-ac59-ed1006d33d36", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.5248065560590476, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:50.244821+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:21:50.245Z" }, { "sequence": 2454, "attempt_id": "c1c658ca-510a-4057-a31a-9bc2c9866d4e", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.6583242990309373, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:50.406745+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T03:21:50.406Z" }, { "sequence": 2455, "attempt_id": "d8baba5e-28e7-4be8-9817-50b4727ab9a8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040185, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.8032652259571478, "model_name": "muse-spark-1.3", "output_tokens": 202, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:50.534930+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0040185, "created_at": "2026-09-17T03:21:50.535Z" }, { "sequence": 2456, "attempt_id": "2fc9e241-0e8f-477d-a648-9ea139cefa05", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003396, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.3088384659495205, "model_name": "gemini-3.1-pro-preview", "output_tokens": 94, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:50.914780+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003396, "created_at": "2026-09-17T03:21:50.914Z" }, { "sequence": 2457, "attempt_id": "aa50f1d4-1e3e-41a5-839c-80f92fb9d758", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.0600382179254666, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:51.525446+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014395, "created_at": "2026-09-17T03:21:51.525Z" }, { "sequence": 2458, "attempt_id": "d2dd6d55-1bc4-46ea-8519-17d5d42d8092", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.9023400950245559, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:51.655627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:21:51.655Z" }, { "sequence": 2459, "attempt_id": "a54792f3-791a-4ac6-bb30-acd141b87bfb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005356, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.5401056739501655, "model_name": "gpt-5.6-terra", "output_tokens": 47, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:52.075348+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005356, "created_at": "2026-09-17T03:21:52.075Z" }, { "sequence": 2460, "attempt_id": "f8f3fd2a-ab57-43b7-8032-ff3c249e895c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.88151810108684, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:52.206640+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:21:52.206Z" }, { "sequence": 2461, "attempt_id": "66d560e1-4e85-4888-9391-1113f2fa889a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.522202803986147, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:52.344430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005758, "created_at": "2026-09-17T03:21:52.344Z" }, { "sequence": 2462, "attempt_id": "e24370fb-9bfa-426f-a3dd-81ff126d0f23", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005943, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 7.769846466020681, "model_name": "muse-spark-1.3", "output_tokens": 651, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:52.466350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005943, "created_at": "2026-09-17T03:21:52.466Z" }, { "sequence": 2463, "attempt_id": "d9d75e26-5a86-4103-a0ae-bcddf3ec472c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.093676015967503, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:52.598513+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:21:52.598Z" }, { "sequence": 2464, "attempt_id": "79ad2fef-1849-4ab5-bdc9-556476a22b7b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.347845523036085, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:52.722731+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T03:21:52.722Z" }, { "sequence": 2465, "attempt_id": "40c55fbf-571c-4d47-9b53-5a22ba8d962f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.9113805490778759, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:52.855222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003707, "created_at": "2026-09-17T03:21:52.856Z" }, { "sequence": 2466, "attempt_id": "90ebd87d-84b4-4af9-9792-e49152a47908", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00159675, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.8314471440389752, "model_name": "gemini-3.8-flash", "output_tokens": 197, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:53.487371+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00159675, "created_at": "2026-09-17T03:21:53.487Z" }, { "sequence": 2467, "attempt_id": "cbd6c397-c8bf-4e9d-a9d7-6181c9142208", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-003", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.397491391049698, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:21:54.253027+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T03:21:54.253Z" }, { "sequence": 2468, "attempt_id": "335f6d66-d80b-4ccb-9753-1e3d5634dd82", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00442625, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 1.8900927560171112, "model_name": "muse-spark-1.2", "output_tokens": 295, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:54.480988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00442625, "created_at": "2026-09-17T03:21:54.481Z" }, { "sequence": 2469, "attempt_id": "c5e1a825-1131-4477-bc70-a65f1dc6f0d5", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006268, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.050002687959932, "model_name": "claude-sonnet-5", "output_tokens": 40, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:54.901732+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006268, "created_at": "2026-09-17T03:21:54.901Z" }, { "sequence": 2470, "attempt_id": "4e762f7e-75ad-4d93-99fa-e9de55610137", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.9518789410358295, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:55.031923+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:21:55.032Z" }, { "sequence": 2471, "attempt_id": "4186f556-4904-4b11-bd15-8b64cefe21fa", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006044, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.510025970987044, "model_name": "gemini-3.1-pro-preview", "output_tokens": 313, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:55.555151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006044, "created_at": "2026-09-17T03:21:55.555Z" }, { "sequence": 2472, "attempt_id": "0841a79d-cf63-47cc-aa7e-b3ce21b94a7f", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016545, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.0830294879851863, "model_name": "claude-opus-5", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:55.688263+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016545, "created_at": "2026-09-17T03:21:55.689Z" }, { "sequence": 2473, "attempt_id": "47e97f67-e5c9-4ef6-8603-138366d3bd51", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006036, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.6269402969628572, "model_name": "gpt-5.6-luna", "output_tokens": 92, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:56.232831+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006036, "created_at": "2026-09-17T03:21:56.232Z" }, { "sequence": 2474, "attempt_id": "da61582c-df55-4c79-8581-d5cbece29c61", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00448575, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 4.049466694006696, "model_name": "muse-spark-1.3", "output_tokens": 309, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:56.365835+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00448575, "created_at": "2026-09-17T03:21:56.366Z" }, { "sequence": 2475, "attempt_id": "4757a8f8-95fd-4fc1-97f2-3cc428b991d7", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.7645284979371354, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:56.501075+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T03:21:56.501Z" }, { "sequence": 2476, "attempt_id": "92abbb60-948c-475b-aa16-5d5caa98e509", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8778869779780507, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:56.633377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:21:56.633Z" }, { "sequence": 2477, "attempt_id": "c9e309f9-8160-4a92-b063-a9b4a982d910", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005988, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.3062102789990604, "model_name": "gpt-5.6-terra", "output_tokens": 88, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:56.761548+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005988, "created_at": "2026-09-17T03:21:56.761Z" }, { "sequence": 2478, "attempt_id": "527d9b68-51c3-4456-88ab-3e43a41cae45", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011424, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.01857228402514, "model_name": "gpt-5.6-sol", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:56.886979+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011424, "created_at": "2026-09-17T03:21:56.887Z" }, { "sequence": 2479, "attempt_id": "0cb36d47-b473-4692-b4e6-84cf340b9081", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.6934561929665506, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:57.710976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001833, "created_at": "2026-09-17T03:21:57.711Z" }, { "sequence": 2480, "attempt_id": "233c31db-199b-4f18-bb8e-e6f4ee996be5", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00227475, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.597528710961342, "model_name": "gemini-3.8-flash", "output_tokens": 364, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:57.844519+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00227475, "created_at": "2026-09-17T03:21:57.846Z" }, { "sequence": 2481, "attempt_id": "29d857c3-3da9-4e18-89c7-9819e0b919af", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.8739139729877934, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:58.635795+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005684, "created_at": "2026-09-17T03:21:58.636Z" }, { "sequence": 2482, "attempt_id": "f525b738-de65-4664-9268-b576635647c6", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 2.0812728740274906, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:58.777332+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01421, "created_at": "2026-09-17T03:21:58.778Z" }, { "sequence": 2483, "attempt_id": "b799594a-373e-4786-b765-39c2eb2d70fa", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.1412852329667658, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:59.119084+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004904, "created_at": "2026-09-17T03:21:59.119Z" }, { "sequence": 2484, "attempt_id": "311b27e2-7486-487f-a8ec-bf17eb484ac4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.1188070620410144, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:59.249284+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02438, "created_at": "2026-09-17T03:21:59.250Z" }, { "sequence": 2485, "attempt_id": "56fe7d87-ced5-436b-8b65-cc36142bb6bb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.724492063978687, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:59.569307+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009752, "created_at": "2026-09-17T03:21:59.569Z" }, { "sequence": 2486, "attempt_id": "87d5f08e-853f-4688-b9a2-4e75d5db04d5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00627625, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 3.6589491240447387, "model_name": "muse-spark-1.2", "output_tokens": 710, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:00.025096+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00627625, "created_at": "2026-09-17T03:22:00.025Z" }, { "sequence": 2487, "attempt_id": "b3308c14-776a-421e-9a22-d9355af42226", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.6673698079539463, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:00.444986+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004904, "created_at": "2026-09-17T03:22:00.445Z" }, { "sequence": 2488, "attempt_id": "0528fa34-08ce-452d-86fa-a36907d40ba4", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 3.8165724920108914, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:00.586849+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02844, "created_at": "2026-09-17T03:22:00.587Z" }, { "sequence": 2489, "attempt_id": "89b786d4-20a3-4908-8d12-b5629077951d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0011175, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.384736948995851, "model_name": "gemini-3.8-flash", "output_tokens": 75, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:00.716426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0011175, "created_at": "2026-09-17T03:22:00.717Z" }, { "sequence": 2490, "attempt_id": "25d31dee-3b60-4441-ac34-a0437ad3e0ca", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003495, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3747723150299862, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:00.844684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003495, "created_at": "2026-09-17T03:22:00.844Z" }, { "sequence": 2491, "attempt_id": "662ade8e-0b5a-485d-9aea-e9efb67c231a", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003502, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.838105597998947, "model_name": "gemini-3.1-pro-preview", "output_tokens": 106, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:01.741249+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003502, "created_at": "2026-09-17T03:22:01.741Z" }, { "sequence": 2492, "attempt_id": "aa1483f3-d00b-4e52-ae79-6100dffb87ef", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00390975, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.7206156300380826, "model_name": "muse-spark-1.2", "output_tokens": 182, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:01.875675+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00390975, "created_at": "2026-09-17T03:22:01.875Z" }, { "sequence": 2493, "attempt_id": "2c71c4c7-daf6-4980-b0e2-e9a24c4f1f90", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.1643199749523774, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:02.136412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:22:02.136Z" }, { "sequence": 2494, "attempt_id": "eb68949c-5c38-47b1-beb6-bc45be9f22ec", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040925, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 2.775418981909752, "model_name": "muse-spark-1.3", "output_tokens": 225, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:02.474627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0040925, "created_at": "2026-09-17T03:22:02.474Z" }, { "sequence": 2495, "attempt_id": "76b05706-34e7-4219-a5ce-7e0d68ed432d", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006248, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 1.920984479947947, "model_name": "claude-sonnet-5", "output_tokens": 57, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:02.765948+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006248, "created_at": "2026-09-17T03:22:02.766Z" }, { "sequence": 2496, "attempt_id": "7f8032d0-642d-4238-856f-3b721740c052", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.113031342974864, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:03.115213+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:22:03.115Z" }, { "sequence": 2497, "attempt_id": "12bf4531-5ff1-481a-a685-52cb196f1020", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.5220339350635186, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:03.245497+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01447, "created_at": "2026-09-17T03:22:03.245Z" }, { "sequence": 2498, "attempt_id": "85d769a9-1792-4ebd-bd30-6004fef4431c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012298, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 25.742531750001945, "model_name": "muse-spark-1.3", "output_tokens": 2141, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:03.648065+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012298, "created_at": "2026-09-17T03:22:03.648Z" }, { "sequence": 2499, "attempt_id": "83001e80-a75e-4456-9bee-3412899b7b92", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005352, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.430087040993385, "model_name": "gpt-5.6-terra", "output_tokens": 46, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:03.777837+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005352, "created_at": "2026-09-17T03:22:03.778Z" }, { "sequence": 2500, "attempt_id": "e245727a-9c3e-49a7-86ee-fac49ce4f1c8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.013166, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.868612103979103, "model_name": "gemini-3.1-pro-preview", "output_tokens": 895, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:03.906407+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.013166, "created_at": "2026-09-17T03:22:03.907Z" }, { "sequence": 2501, "attempt_id": "8a807286-40a1-4f38-bbb6-499110f1add1", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.067502925056033, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:04.046926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:22:04.047Z" }, { "sequence": 2502, "attempt_id": "0a89a02f-aac9-4d3c-a0c6-41667f348e6b", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.8169172019697726, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:04.191563+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:22:04.191Z" }, { "sequence": 2503, "attempt_id": "71296aa0-2c57-496f-a7bd-c3bc5ffeee6d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.6302618640474975, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:04.318605+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:22:04.318Z" }, { "sequence": 2504, "attempt_id": "e3e80ea6-71c0-4de9-8d54-a395f7b73dbf", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00148875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.4228718189988285, "model_name": "gemini-3.8-flash", "output_tokens": 167, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:04.668654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00148875, "created_at": "2026-09-17T03:22:04.668Z" }, { "sequence": 2505, "attempt_id": "ced0a6c3-c6e3-468c-ae55-114c13f4d289", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000576, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.1838940989691764, "model_name": "gpt-5.6-luna", "output_tokens": 80, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:04.798834+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000576, "created_at": "2026-09-17T03:22:04.800Z" }, { "sequence": 2506, "attempt_id": "1d977ec5-c1f4-4f6e-95d8-a6c7d9be2e4b", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.7179965369869024, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:05.162398+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T03:22:05.162Z" }, { "sequence": 2507, "attempt_id": "b9c37474-efca-4c10-99fa-d5fd2142981f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003908, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.601071208016947, "model_name": "gemini-3.1-pro-preview", "output_tokens": 134, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:05.495396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003908, "created_at": "2026-09-17T03:22:05.495Z" }, { "sequence": 2508, "attempt_id": "b3c45010-4fba-45e5-8837-9d9d774bfe64", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.6735792710678652, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:05.992486+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00601, "created_at": "2026-09-17T03:22:05.992Z" }, { "sequence": 2509, "attempt_id": "8f3c4208-c22b-4a69-8bec-4e694d1e6e7e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.0277729900553823, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:06.202801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015025, "created_at": "2026-09-17T03:22:06.202Z" }, { "sequence": 2510, "attempt_id": "8ab6d635-f760-4c9e-9e53-02f4b9a50d67", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.5427031599683687, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:06.341794+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:22:06.341Z" }, { "sequence": 2511, "attempt_id": "f8351f71-599c-4919-9324-7088e8005f8d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005358, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.2566830880241469, "model_name": "gpt-5.6-terra", "output_tokens": 36, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:06.560709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005358, "created_at": "2026-09-17T03:22:06.560Z" }, { "sequence": 2512, "attempt_id": "570ded74-8df1-4ba8-b65b-99dc76e3eda3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.9945315150544047, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:06.922984+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:22:06.923Z" }, { "sequence": 2513, "attempt_id": "4481afeb-1bcd-4e00-b049-b790ab19a4b2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005466000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.512335438048467, "model_name": "gpt-5.6-luna", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:07.137080+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005466000000000001, "created_at": "2026-09-17T03:22:07.137Z" }, { "sequence": 2514, "attempt_id": "7210efb1-7a12-4327-8d84-36dd389f374b", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.8524426079820842, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:07.323045+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:22:07.323Z" }, { "sequence": 2515, "attempt_id": "1adffe89-5a02-4f4e-9a06-6e8dd54dbfce", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001335, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.2954809730872512, "model_name": "gemini-3.8-flash", "output_tokens": 114, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:07.626770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001335, "created_at": "2026-09-17T03:22:07.626Z" }, { "sequence": 2516, "attempt_id": "cefa63a3-693e-4478-a371-a554ca88b73d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0059345, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.0798594179796055, "model_name": "muse-spark-1.2", "output_tokens": 649, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:07.986542+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0059345, "created_at": "2026-09-17T03:22:07.986Z" }, { "sequence": 2517, "attempt_id": "26227cbe-a182-4903-9695-0c7964072c3b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.007874668925069, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:08.116881+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:22:08.117Z" }, { "sequence": 2518, "attempt_id": "03471440-7241-4b9b-a30b-2944d544edd5", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.5818098959280178, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:22:08.698996+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T03:22:08.699Z" }, { "sequence": 2519, "attempt_id": "be2d9ad5-89e0-4df7-b228-5650b0dda103", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.047519780928269, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:09.804821+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00601, "created_at": "2026-09-17T03:22:09.804Z" }, { "sequence": 2520, "attempt_id": "397de3ee-bddc-4e1c-bc93-31610b37c60e", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00488, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.702692796010524, "model_name": "gemini-3.1-pro-preview", "output_tokens": 205, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:09.939296+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00488, "created_at": "2026-09-17T03:22:09.939Z" }, { "sequence": 2521, "attempt_id": "03a4ba9d-167b-4531-a46b-32238d9aff21", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004918, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 3.551690642023459, "model_name": "muse-spark-1.3", "output_tokens": 391, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:10.243742+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004918, "created_at": "2026-09-17T03:22:10.243Z" }, { "sequence": 2522, "attempt_id": "6ab6b10e-6a56-4f1e-8f96-d88dd86670a6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00490525, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 3.3230832429835573, "model_name": "muse-spark-1.2", "output_tokens": 388, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:10.389536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00490525, "created_at": "2026-09-17T03:22:10.389Z" }, { "sequence": 2523, "attempt_id": "dcc5b2c5-d70e-480f-9ef6-3ff4b3c7a249", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00911525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 14.76292517199181, "model_name": "muse-spark-1.3", "output_tokens": 1378, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:10.580893+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00911525, "created_at": "2026-09-17T03:22:10.581Z" }, { "sequence": 2524, "attempt_id": "2bf51c75-3dca-48a6-8fb8-95345789b5f1", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.5823104990413412, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:10.839957+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:22:10.840Z" }, { "sequence": 2525, "attempt_id": "8b872019-5bb2-495d-87b7-440bfcd2301f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.040670281043276, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:22:11.304880+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T03:22:11.305Z" }, { "sequence": 2526, "attempt_id": "d897f535-9787-4024-83ae-aed6fb6a1146", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018125, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.970839538029395, "model_name": "claude-opus-5", "output_tokens": 135, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:11.453048+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.018125, "created_at": "2026-09-17T03:22:11.453Z" }, { "sequence": 2527, "attempt_id": "5044b347-f806-4d49-9559-22dc54288752", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 3.1095218689879403, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:11.940891+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:22:11.941Z" }, { "sequence": 2528, "attempt_id": "53baad7a-401d-4a81-9048-f8592d7dd982", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005382, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.129191461019218, "model_name": "gpt-5.6-terra", "output_tokens": 38, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:12.074638+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005382, "created_at": "2026-09-17T03:22:12.074Z" }, { "sequence": 2529, "attempt_id": "bccf1c70-c7ec-4cce-afcc-2c1ab56148c9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.2992050889879465, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:22:12.200145+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:22:12.200Z" }, { "sequence": 2530, "attempt_id": "3152d666-cf81-4d53-93db-c5ef1c8f71f2", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00144, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.6664817900164053, "model_name": "gemini-3.8-flash", "output_tokens": 142, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:22:12.331448+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00144, "created_at": "2026-09-17T03:22:12.331Z" }, { "sequence": 2531, "attempt_id": "5e58caf9-0b2a-411b-8d0f-04495ae9cd2e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0065295, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 9.013796739047393, "model_name": "muse-spark-1.3", "output_tokens": 789, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:12.791926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0065295, "created_at": "2026-09-17T03:22:12.792Z" }, { "sequence": 2532, "attempt_id": "68402b91-3e41-4180-ae37-89c994e8313c", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.5867185749812052, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:12.923730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T03:22:12.923Z" }, { "sequence": 2533, "attempt_id": "a899cc1b-b2b0-4f3c-b183-05559c9577f9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005946, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 3.392854274949059, "model_name": "gpt-5.6-luna", "output_tokens": 85, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:13.461373+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005946, "created_at": "2026-09-17T03:22:13.461Z" }, { "sequence": 2534, "attempt_id": "0ab7769b-4146-4c7e-a10f-316a913b1e7b", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004652, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.4063302790746093, "model_name": "gemini-3.1-pro-preview", "output_tokens": 186, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:22:13.792226+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004652, "created_at": "2026-09-17T03:22:13.792Z" }, { "sequence": 2535, "attempt_id": "6ceea81e-e765-408d-97f6-ed30ec93ffd3", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.6927483410108835, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:14.020576+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T03:22:14.020Z" }, { "sequence": 2536, "attempt_id": "fc42db45-f558-4c61-aedd-e14ab61697c3", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.8571833278983831, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:14.330166+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:22:14.330Z" }, { "sequence": 2537, "attempt_id": "c3e5c055-ea00-446c-be61-e1239033c7b8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00480325, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 3.486556390998885, "model_name": "muse-spark-1.3", "output_tokens": 364, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:22:14.463275+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00480325, "created_at": "2026-09-17T03:22:14.464Z" }, { "sequence": 2538, "attempt_id": "3ca7f177-32e9-45fa-92ed-274140d1242c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.560659519978799, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:14.592870+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:22:14.594Z" }, { "sequence": 2539, "attempt_id": "f6014e8d-98d2-4d80-8306-b253cdef1bb2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005476, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6088955219602212, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:14.723663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005476, "created_at": "2026-09-17T03:22:14.723Z" }, { "sequence": 2540, "attempt_id": "2208e5aa-3021-4fb0-88b0-e614efd5b676", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01584, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.8614589099306613, "model_name": "claude-opus-5", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:14.936378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01584, "created_at": "2026-09-17T03:22:14.936Z" }, { "sequence": 2541, "attempt_id": "9a1bc97c-17fb-48a7-9897-c4dd506c6a71", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.837022946914658, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:15.300580+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003561, "created_at": "2026-09-17T03:22:15.300Z" }, { "sequence": 2542, "attempt_id": "66f1c9d5-797b-491a-b998-de788ba28b56", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00475225, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.306712856981903, "model_name": "muse-spark-1.2", "output_tokens": 352, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:22:15.760083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00475225, "created_at": "2026-09-17T03:22:15.760Z" }, { "sequence": 2543, "attempt_id": "1e65ec59-19dc-4009-b4c3-6397c32907f0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.292574310908094, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:15.893391+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:22:15.893Z" }, { "sequence": 2544, "attempt_id": "f362ee1d-a067-4432-be4a-0af3c87a156f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.7185355110559613, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:16.612244+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T03:22:16.612Z" }, { "sequence": 2545, "attempt_id": "5da91075-ced4-43be-8d6c-e742b8c8b81b", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00351525, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.874995445017703, "model_name": "gemini-3.8-flash", "output_tokens": 710, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:17.036549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00351525, "created_at": "2026-09-17T03:22:17.036Z" }, { "sequence": 2546, "attempt_id": "fb66f427-1cbe-4f25-91a9-a8722f0e07cb", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005728, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 1.7311993250623345, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:17.169736+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005728, "created_at": "2026-09-17T03:22:17.169Z" }, { "sequence": 2547, "attempt_id": "0d9d54f4-e5ad-45f7-8e95-53f3f1029dbd", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005214, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.369887802982703, "model_name": "gemini-3.1-pro-preview", "output_tokens": 245, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:17.300730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005214, "created_at": "2026-09-17T03:22:17.300Z" }, { "sequence": 2548, "attempt_id": "43c0e1aa-c7b6-4d49-9046-4e0ea22e0fd6", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 6.21435724501498, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:17.812363+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T03:22:17.812Z" }, { "sequence": 2549, "attempt_id": "98628767-7173-40c4-8359-171246a196b9", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01537, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.795630228938535, "model_name": "claude-opus-5", "output_tokens": 53, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:17.947336+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01537, "created_at": "2026-09-17T03:22:17.947Z" }, { "sequence": 2550, "attempt_id": "cc0698f9-4b18-4d9d-bf91-415391c151d6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.159025999950245, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:18.194868+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:22:18.195Z" }, { "sequence": 2551, "attempt_id": "9fead7dd-6a1b-4e3b-af4c-8a4acef1bff1", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.5113138390006498, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:18.377474+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T03:22:18.377Z" }, { "sequence": 2552, "attempt_id": "ee4a3582-57dd-4e2f-b0d8-2dfd82ed5dde", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010692, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.8309030949603766, "model_name": "gpt-5.6-sol", "output_tokens": 57, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:18.580400+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010692, "created_at": "2026-09-17T03:22:18.580Z" }, { "sequence": 2553, "attempt_id": "6115a41f-bea0-4086-a54b-dc0fa9b27e8c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005328, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.652316435938701, "model_name": "gpt-5.6-terra", "output_tokens": 46, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:18.822344+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005328, "created_at": "2026-09-17T03:22:18.822Z" }, { "sequence": 2554, "attempt_id": "bb2389aa-44da-463c-8e7e-ee9178dbdef8", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.8176993959350511, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:18.955193+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000368, "created_at": "2026-09-17T03:22:18.955Z" }, { "sequence": 2555, "attempt_id": "e99cf100-3947-48c6-8244-595eaaa27178", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.7791079480666667, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:19.086563+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004944, "created_at": "2026-09-17T03:22:19.086Z" }, { "sequence": 2556, "attempt_id": "3d86bea7-8314-46e5-9d66-458eecb61e01", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.646148188970983, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:19.864552+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T03:22:19.864Z" }, { "sequence": 2557, "attempt_id": "b3f5d573-6dd9-471f-bbee-437eaf47a0cd", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002055, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 2.436765832011588, "model_name": "gemini-3.8-flash", "output_tokens": 321, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:20.384419+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002055, "created_at": "2026-09-17T03:22:20.384Z" }, { "sequence": 2558, "attempt_id": "89d0a135-8b25-4585-977d-f1a5742df906", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006348, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.027626106981188, "model_name": "claude-sonnet-5", "output_tokens": 67, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:21.114515+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006348, "created_at": "2026-09-17T03:22:21.114Z" }, { "sequence": 2559, "attempt_id": "1a64d6c6-c83e-434a-8dca-dbabf5c7a8f6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00490375, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 2.8059691989328712, "model_name": "muse-spark-1.2", "output_tokens": 410, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:21.316702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00490375, "created_at": "2026-09-17T03:22:21.316Z" }, { "sequence": 2560, "attempt_id": "74582cfc-0640-4556-a106-601bcd7af6f5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00791525, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 6.787124107941054, "model_name": "muse-spark-1.2", "output_tokens": 1118, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:21.511062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00791525, "created_at": "2026-09-17T03:22:21.511Z" }, { "sequence": 2561, "attempt_id": "bfd25216-5416-4f1f-9381-39cb682eee52", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.061561333015561, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:21.655618+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:22:21.655Z" }, { "sequence": 2562, "attempt_id": "f36c9f8f-1a4e-4495-8ab4-46245f268b6f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00489525, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 3.254211710067466, "model_name": "muse-spark-1.3", "output_tokens": 408, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:21.784105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00489525, "created_at": "2026-09-17T03:22:21.784Z" }, { "sequence": 2563, "attempt_id": "6456f4b8-7f2b-42f2-b641-f33158a823f5", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9335888529894873, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:21.943614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:22:21.943Z" }, { "sequence": 2564, "attempt_id": "7879827d-461e-4ba9-8701-324f2eb993dc", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7466606019297615, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:22.665712+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:22:22.665Z" }, { "sequence": 2565, "attempt_id": "d241ca25-13a5-4480-a41e-70831d04b803", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.732013701926917, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:22.814274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01447, "created_at": "2026-09-17T03:22:22.815Z" }, { "sequence": 2566, "attempt_id": "047f1adb-04aa-4225-a6aa-057f02c90c54", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00534, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.5432474220870063, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:22.940817+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00534, "created_at": "2026-09-17T03:22:22.940Z" }, { "sequence": 2567, "attempt_id": "977dc55f-081f-406d-87d8-af04ca599343", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 4.351046836003661, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:23.074725+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T03:22:23.076Z" }, { "sequence": 2568, "attempt_id": "6d914ba0-644e-47ce-afd5-cc0a99ce7295", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008654, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.9708767760312185, "model_name": "gemini-3.1-pro-preview", "output_tokens": 532, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:23.413445+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008654, "created_at": "2026-09-17T03:22:23.413Z" }, { "sequence": 2569, "attempt_id": "852068da-46dd-4aa0-8fe5-010760fe5664", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0058285, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 8.939542611013167, "model_name": "muse-spark-1.3", "output_tokens": 627, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:23.547978+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0058285, "created_at": "2026-09-17T03:22:23.548Z" }, { "sequence": 2570, "attempt_id": "6a229927-6156-41b3-8d60-14580f2aaaab", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00191625, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.8589035790646449, "model_name": "gemini-3.8-flash", "output_tokens": 281, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:23.774141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00191625, "created_at": "2026-09-17T03:22:23.774Z" }, { "sequence": 2571, "attempt_id": "a5faac0e-2fad-46f7-a834-2d2e2c43bbbf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.1843049729941413, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:24.632858+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004968, "created_at": "2026-09-17T03:22:24.633Z" }, { "sequence": 2572, "attempt_id": "8e057fbb-1bf8-464d-a9d7-fa471bc99807", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.2578327850205824, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:24.806095+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:22:24.806Z" }, { "sequence": 2573, "attempt_id": "d605379b-58ea-4fd1-a1b5-e530db6bed0c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7225757649866864, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:24.938181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:22:24.938Z" }, { "sequence": 2574, "attempt_id": "1fdcd2e2-1707-4e1c-887e-4ce389790deb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.184689178946428, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:25.091422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:22:25.091Z" }, { "sequence": 2575, "attempt_id": "433ddebf-25f5-4907-8f89-027ff342a4eb", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.3435546460095793, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:25.418550+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:22:25.418Z" }, { "sequence": 2576, "attempt_id": "bc66fe62-fe5d-47a3-a7fd-26617eedf694", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7723401909461245, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:25.552651+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:22:25.552Z" }, { "sequence": 2577, "attempt_id": "c3391425-772d-46bd-bcdb-bea2d94117a8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00519925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 3.240561430924572, "model_name": "muse-spark-1.2", "output_tokens": 476, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:26.055121+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00519925, "created_at": "2026-09-17T03:22:26.055Z" }, { "sequence": 2578, "attempt_id": "cc922234-ab37-43a5-a134-562b8664d16b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.189512267941609, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:26.201104+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:22:26.201Z" }, { "sequence": 2579, "attempt_id": "6b6dacf1-1571-453c-a17b-602285951b24", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.6507346229627728, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:26.333678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:22:26.333Z" }, { "sequence": 2580, "attempt_id": "c180e4ac-81b6-474c-8e3f-64a2e0fe3270", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012105, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.653496494051069, "model_name": "gemini-3.8-flash", "output_tokens": 96, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:26.874164+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0012105, "created_at": "2026-09-17T03:22:26.874Z" }, { "sequence": 2581, "attempt_id": "3995cfef-2fda-4dd1-b708-84c36bbbcb5f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007616, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 5.348469171090983, "model_name": "gemini-3.1-pro-preview", "output_tokens": 443, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:27.010774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007616, "created_at": "2026-09-17T03:22:27.011Z" }, { "sequence": 2582, "attempt_id": "43b76b64-1506-4488-8651-b20b817b4976", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.1357052329694852, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:27.151690+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:22:27.151Z" }, { "sequence": 2583, "attempt_id": "8419ff97-6ce5-4e31-880b-830cb1eb773c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003636, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.5322925719665363, "model_name": "gemini-3.1-pro-preview", "output_tokens": 114, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:27.614599+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003636, "created_at": "2026-09-17T03:22:27.614Z" }, { "sequence": 2584, "attempt_id": "6ea3ae39-add4-4209-b5dd-d7c0498e9a81", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.6986314459936693, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:27.980176+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T03:22:27.980Z" }, { "sequence": 2585, "attempt_id": "5cb99078-0364-41aa-b79c-d4c984f5f7af", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.862789642997086, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:28.328902+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014395, "created_at": "2026-09-17T03:22:28.329Z" }, { "sequence": 2586, "attempt_id": "dde1830c-18c8-4d6f-8f31-bc5ec3cf3af2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004027, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.6606194220948964, "model_name": "muse-spark-1.2", "output_tokens": 204, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:28.862024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004027, "created_at": "2026-09-17T03:22:28.862Z" }, { "sequence": 2587, "attempt_id": "612f826e-9255-4295-b136-a24d85aaca5c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0045115, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.312083658995107, "model_name": "muse-spark-1.3", "output_tokens": 318, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:28.997929+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0045115, "created_at": "2026-09-17T03:22:28.998Z" }, { "sequence": 2588, "attempt_id": "7827caea-6f39-4dde-abc5-57712baa7057", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.415822479990311, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:29.167025+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:22:29.167Z" }, { "sequence": 2589, "attempt_id": "0f31af88-3be1-4ff8-9123-3b5f9f4f86d8", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006488, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.2246478599263355, "model_name": "claude-sonnet-5", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:29.301614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006488, "created_at": "2026-09-17T03:22:29.301Z" }, { "sequence": 2590, "attempt_id": "db2285d2-729e-4ce9-87ed-ea6c5bc7f3b1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.9535656590014696, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:29.436732+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:22:29.436Z" }, { "sequence": 2591, "attempt_id": "358a8c7a-597a-4f21-a249-4c18299a96e8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005236, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.2306192690739408, "model_name": "gpt-5.6-terra", "output_tokens": 37, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:29.578069+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005236, "created_at": "2026-09-17T03:22:29.578Z" }, { "sequence": 2592, "attempt_id": "73a465df-3365-4b86-af11-3027c0a0eb0d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 6.338730292976834, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:29.706150+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:22:29.706Z" }, { "sequence": 2593, "attempt_id": "274f6788-b18c-4f95-b486-7aee57f22b58", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.3245373900281265, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:29.843751+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:22:29.845Z" }, { "sequence": 2594, "attempt_id": "732f88e5-3495-4dd6-98f7-dc434369e735", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0056625, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 7.7919144129846245, "model_name": "muse-spark-1.3", "output_tokens": 585, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:29.977826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0056625, "created_at": "2026-09-17T03:22:29.979Z" }, { "sequence": 2595, "attempt_id": "07054a49-0374-4560-a33d-6303057b0a2a", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.9634422870585695, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:30.265348+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003707, "created_at": "2026-09-17T03:22:30.265Z" }, { "sequence": 2596, "attempt_id": "2a48d21a-97cf-46be-96b0-73f3d242db61", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 4.1238349709892645, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:30.457790+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T03:22:30.457Z" }, { "sequence": 2597, "attempt_id": "683c309c-ebbe-484e-b096-cdb134abd8f4", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0014505, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 2.0745261419797316, "model_name": "gemini-3.8-flash", "output_tokens": 158, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:31.205165+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0014505, "created_at": "2026-09-17T03:22:31.205Z" }, { "sequence": 2598, "attempt_id": "f532c2b4-12ba-48a7-af2a-311c2ed33b3c", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 1.2464016140438616, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:31.369785+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001884, "created_at": "2026-09-17T03:22:31.369Z" }, { "sequence": 2599, "attempt_id": "6e686f2c-94a7-43d0-98e1-7ddf31ebfdca", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00429875, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.5358600839972496, "model_name": "muse-spark-1.2", "output_tokens": 265, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:32.371798+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00429875, "created_at": "2026-09-17T03:22:32.371Z" }, { "sequence": 2600, "attempt_id": "0a88fe48-83f1-486d-aa58-2d404e9608a4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004688, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 3.6450624089920893, "model_name": "gemini-3.1-pro-preview", "output_tokens": 200, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:32.643272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004688, "created_at": "2026-09-17T03:22:32.643Z" }, { "sequence": 2601, "attempt_id": "c2477d13-6c98-4c0a-8917-bf5793441701", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00467275, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.522048045997508, "model_name": "muse-spark-1.3", "output_tokens": 353, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:32.959076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00467275, "created_at": "2026-09-17T03:22:32.959Z" }, { "sequence": 2602, "attempt_id": "1a0a3bf3-8309-4626-b2c7-c8d018c60e49", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01759, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.244338770979084, "model_name": "claude-opus-5", "output_tokens": 130, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:33.093863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01759, "created_at": "2026-09-17T03:22:33.094Z" }, { "sequence": 2603, "attempt_id": "c9595490-beeb-49ba-80c6-9d03b70a72bf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005824, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6689251499483362, "model_name": "gpt-5.6-luna", "output_tokens": 83, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:33.241247+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005824, "created_at": "2026-09-17T03:22:33.241Z" }, { "sequence": 2604, "attempt_id": "fd424cd4-9542-47c7-b7f7-1e690fa45276", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008126, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.2664555419469252, "model_name": "claude-sonnet-5", "output_tokens": 239, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:33.374710+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008126, "created_at": "2026-09-17T03:22:33.374Z" }, { "sequence": 2605, "attempt_id": "66b07302-ab3d-4a4c-9260-b8cccade2d25", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02925, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 3.665623558103107, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:33.511553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02925, "created_at": "2026-09-17T03:22:33.511Z" }, { "sequence": 2606, "attempt_id": "e2737946-45eb-4a16-838e-848e9eb8ed2e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011796, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.9130825269967318, "model_name": "gpt-5.6-sol", "output_tokens": 107, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:33.640927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011796, "created_at": "2026-09-17T03:22:33.641Z" }, { "sequence": 2607, "attempt_id": "101ebff8-ed2c-4b2c-b068-de54fd7cc5c8", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.7040724960388616, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:33.798211+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003761, "created_at": "2026-09-17T03:22:33.798Z" }, { "sequence": 2608, "attempt_id": "4595fb24-dcc6-4e5f-9928-b65229b79f2a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02639, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 3.769161332049407, "model_name": "gpt-6-astra", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:34.169117+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02639, "created_at": "2026-09-17T03:22:34.169Z" }, { "sequence": 2609, "attempt_id": "5d849b9f-c2c6-4d5d-b8dd-1b7e37618404", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6354702030075714, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:34.567063+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:22:34.567Z" }, { "sequence": 2610, "attempt_id": "671cbec2-efbf-40b7-8470-635b99ddf5b2", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.6208739780122414, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:35.410610+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:22:35.410Z" }, { "sequence": 2611, "attempt_id": "ce5b29a3-9ca6-40ef-beeb-123014ae5f96", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005704, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 4.1431485519278795, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:35.559927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005704, "created_at": "2026-09-17T03:22:35.561Z" }, { "sequence": 2612, "attempt_id": "a39ed5ec-1278-42b6-a4fd-2ba4ec379091", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.8594985419185832, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:35.696280+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:22:35.696Z" }, { "sequence": 2613, "attempt_id": "bcd33b21-79ea-4b64-b985-6a55a451857a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.8864852610277012, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:36.190656+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:22:36.190Z" }, { "sequence": 2614, "attempt_id": "d0af447d-59ec-4858-b6a5-7d8ade0c720d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1145358890062198, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:36.674773+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:22:36.674Z" }, { "sequence": 2615, "attempt_id": "243d8dad-b4ac-4875-879b-400e9aa8064f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.2746495408937335, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:36.971222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:22:36.971Z" }, { "sequence": 2616, "attempt_id": "0df7299e-1a57-4d48-a36d-8c8a37746f5e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005454, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 4.546380468993448, "model_name": "gemini-3.8-flash", "output_tokens": 1222, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:37.324622+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005454, "created_at": "2026-09-17T03:22:37.324Z" }, { "sequence": 2617, "attempt_id": "4857899c-2eb8-4322-9015-830b41a52280", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.9579442810500041, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:37.784195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:22:37.784Z" }, { "sequence": 2618, "attempt_id": "e0b3407e-8435-4d5f-b020-9be351ca74d0", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.648759257979691, "model_name": "gemini-3.8-flash", "output_tokens": 119, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:37.976359+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00129675, "created_at": "2026-09-17T03:22:37.976Z" }, { "sequence": 2619, "attempt_id": "19c0be71-1aaa-4483-bc0d-1efd938afb52", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003372, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.8001735650468618, "model_name": "gemini-3.1-pro-preview", "output_tokens": 92, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:38.630834+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003372, "created_at": "2026-09-17T03:22:38.631Z" }, { "sequence": 2620, "attempt_id": "7b82e422-8c7b-461c-90ea-d4909aad1158", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.010256, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 6.335312629933469, "model_name": "gemini-3.1-pro-preview", "output_tokens": 661, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:38.845052+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010256, "created_at": "2026-09-17T03:22:38.845Z" }, { "sequence": 2621, "attempt_id": "433e7636-531a-4bd5-acb4-76ad81a9de57", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00419275, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.8391146209323779, "model_name": "muse-spark-1.2", "output_tokens": 243, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:39.302477+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00419275, "created_at": "2026-09-17T03:22:39.302Z" }, { "sequence": 2622, "attempt_id": "162b7806-7c1d-4ef7-abdd-5c14389cc774", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.803821467095986, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:39.783832+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T03:22:39.783Z" }, { "sequence": 2623, "attempt_id": "025f441e-a591-48b8-9ac2-d8d48718b56b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00460925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.792696001008153, "model_name": "muse-spark-1.3", "output_tokens": 341, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:39.923521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00460925, "created_at": "2026-09-17T03:22:39.923Z" }, { "sequence": 2624, "attempt_id": "09257555-e27f-4962-a00f-0bb3c9a18ab7", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.8035372260492295, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:40.582801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00571, "created_at": "2026-09-17T03:22:40.582Z" }, { "sequence": 2625, "attempt_id": "a5edbbeb-0eb9-444d-ad9f-dc319f32a026", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 6.266449699993245, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:40.970297+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:22:40.970Z" }, { "sequence": 2626, "attempt_id": "4dec9258-0ea6-4f75-8041-e2a8faa7e98e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 3.1574304359965026, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:41.272068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014275, "created_at": "2026-09-17T03:22:41.272Z" }, { "sequence": 2627, "attempt_id": "3f180ce3-9c4c-4e19-91ac-6ec71ac77d34", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.9710747170029208, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:41.411370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:22:41.411Z" }, { "sequence": 2628, "attempt_id": "bf4ca031-25f3-44ed-94c4-126f927d740a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.7629120759665966, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:41.686728+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:22:41.686Z" }, { "sequence": 2629, "attempt_id": "3963a5aa-c14c-421b-96ec-e62849437c27", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8563484740443528, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:42.414809+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:22:42.415Z" }, { "sequence": 2630, "attempt_id": "37eed5e2-9a4d-4b05-8c50-a95d842fc0ed", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005288, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.7199146290076897, "model_name": "gpt-5.6-terra", "output_tokens": 44, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:42.791295+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005288, "created_at": "2026-09-17T03:22:42.791Z" }, { "sequence": 2631, "attempt_id": "cbf85799-dc3c-4a2e-8e15-60e3be1a9869", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.143947137054056, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:42.929271+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:22:42.929Z" }, { "sequence": 2632, "attempt_id": "dae09e77-0cc8-4faf-bb3a-87cbbe09e702", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001494, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 2.138473421917297, "model_name": "gemini-3.8-flash", "output_tokens": 173, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:43.547873+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001494, "created_at": "2026-09-17T03:22:43.548Z" }, { "sequence": 2633, "attempt_id": "a9db81ac-b29e-4c96-82be-665c310fe289", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 5.866790223051794, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:43.790216+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02857, "created_at": "2026-09-17T03:22:43.790Z" }, { "sequence": 2634, "attempt_id": "c3bfa441-d055-4af8-897b-e618339e98e7", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004618, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 3.3396139569813386, "model_name": "gemini-3.1-pro-preview", "output_tokens": 197, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:44.447657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004618, "created_at": "2026-09-17T03:22:44.447Z" }, { "sequence": 2635, "attempt_id": "ac3afbcf-b6b2-49b1-9b9c-660b31157a7a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.7692851269384846, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:44.696589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T03:22:44.696Z" }, { "sequence": 2636, "attempt_id": "4fd9958c-879d-480e-868a-96ed56fa5588", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004436, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 2.9013583039632067, "model_name": "muse-spark-1.3", "output_tokens": 302, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:44.845715+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004436, "created_at": "2026-09-17T03:22:44.845Z" }, { "sequence": 2637, "attempt_id": "c9ec430c-b679-4ac2-9fae-9d44a1535579", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004181, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 2.4464185580145568, "model_name": "muse-spark-1.2", "output_tokens": 242, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:45.002629+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004181, "created_at": "2026-09-17T03:22:45.002Z" }, { "sequence": 2638, "attempt_id": "02be0905-034e-4341-8081-4073661016e0", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.6579577049706131, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:45.344487+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T03:22:45.344Z" }, { "sequence": 2639, "attempt_id": "ae3d449d-b1f1-48c9-89b8-61d3bac67059", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.7649473830824718, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:46.352231+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:22:46.352Z" }, { "sequence": 2640, "attempt_id": "0a6f1937-bad7-4abf-8141-31a5d20b4446", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.642505026073195, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:46.779515+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:22:46.779Z" }, { "sequence": 2641, "attempt_id": "6c5dd3fa-333c-4429-b5bc-af0ef507b989", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.067391016986221, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:46.920991+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:22:46.921Z" }, { "sequence": 2642, "attempt_id": "abc9908e-4dc5-4719-a71b-bc2e7b90d972", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.264705620938912, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:47.247480+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:22:47.247Z" }, { "sequence": 2643, "attempt_id": "acabd9d9-013a-4d01-ad39-42c8e753ff85", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 4.340433359029703, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:47.455412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T03:22:47.455Z" }, { "sequence": 2644, "attempt_id": "aabb7ee0-f70b-4425-81ea-00ef85e9c3ba", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00130875, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.1282771669793874, "model_name": "gemini-3.8-flash", "output_tokens": 126, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:47.649890+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00130875, "created_at": "2026-09-17T03:22:47.650Z" }, { "sequence": 2645, "attempt_id": "3a08cf0c-b88a-4a0b-a199-2e6922cb043d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 4.446110393968411, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:47.800568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T03:22:47.800Z" }, { "sequence": 2646, "attempt_id": "30e0586e-a190-4b23-859d-427b7ef09ee3", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 14.002781323972158, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:47.942054+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:22:47.942Z" }, { "sequence": 2647, "attempt_id": "5e157680-b280-465a-ab71-506cc0829d7c", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.998752381070517, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:48.085649+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:22:48.085Z" }, { "sequence": 2648, "attempt_id": "57830d2b-591f-49fc-8d7b-b3f4bdbb9dca", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.7625779110239819, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:48.848551+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:22:48.848Z" }, { "sequence": 2649, "attempt_id": "39d059c2-1ea3-41c4-a1ff-13ec84109f6c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.446990331984125, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:49.528710+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:22:49.528Z" }, { "sequence": 2650, "attempt_id": "2488e053-d050-4f1b-a5cc-0c1e5521e29f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003903, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.14749312994536, "model_name": "muse-spark-1.2", "output_tokens": 181, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:49.669159+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003903, "created_at": "2026-09-17T03:22:49.669Z" }, { "sequence": 2651, "attempt_id": "410b37d7-ad67-45a7-a8ea-b4399742161c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00523, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.36672475899104, "model_name": "gemini-3.1-pro-preview", "output_tokens": 250, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:49.850538+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00523, "created_at": "2026-09-17T03:22:49.850Z" }, { "sequence": 2652, "attempt_id": "7c3d602c-2a65-413c-9cb8-037b3530b4cf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.2066786859650165, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:50.206721+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:22:50.206Z" }, { "sequence": 2653, "attempt_id": "a1cc32d7-49e9-43e0-924a-6223c566d0d2", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.152431872091256, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:50.358358+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:22:50.358Z" }, { "sequence": 2654, "attempt_id": "c0c65e79-ff53-4be5-bc38-4f410aae830a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.4017998420167714, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:50.623313+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:22:50.623Z" }, { "sequence": 2655, "attempt_id": "171bc9f0-4a29-48e9-b33f-ba0203c42869", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00428975, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.601227528997697, "model_name": "muse-spark-1.3", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:50.763050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00428975, "created_at": "2026-09-17T03:22:50.764Z" }, { "sequence": 2656, "attempt_id": "f07f4833-249a-455f-bbf2-6d0a72cfb6a6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0873682150850073, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:50.896389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:22:50.896Z" }, { "sequence": 2657, "attempt_id": "f2eb5275-1853-4f0e-a3d3-d54226e12e8d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.8426596929784864, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:51.041443+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:22:51.041Z" }, { "sequence": 2658, "attempt_id": "8107fb49-eca9-4a6b-acfc-91fa67fd9116", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7659274289617315, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:51.261268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:22:51.261Z" }, { "sequence": 2659, "attempt_id": "74a3b787-268d-4d12-9483-03d805e6e651", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 4.0052507689688355, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:51.655455+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:22:51.655Z" }, { "sequence": 2660, "attempt_id": "9461be09-edeb-4c22-8f1f-7df464aae117", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00123675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.9848807089729235, "model_name": "gemini-3.8-flash", "output_tokens": 103, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:52.343564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00123675, "created_at": "2026-09-17T03:22:52.343Z" }, { "sequence": 2661, "attempt_id": "58b1c783-9f44-44ba-ad35-c1add27f39d6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041035, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.5774850669549778, "model_name": "muse-spark-1.2", "output_tokens": 222, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:52.754012+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0041035, "created_at": "2026-09-17T03:22:52.754Z" }, { "sequence": 2662, "attempt_id": "22eb4a4b-72dc-472e-8f5c-6956087ec958", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004068, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.7862808789359406, "model_name": "gemini-3.1-pro-preview", "output_tokens": 150, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:52.897507+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004068, "created_at": "2026-09-17T03:22:52.897Z" }, { "sequence": 2663, "attempt_id": "77db33f7-8d61-45eb-b1e5-3953964dfa14", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6399901380063966, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:53.040276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:22:53.040Z" }, { "sequence": 2664, "attempt_id": "04a856bb-144a-4179-b9b3-3b1459b4a185", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.401790733099915, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:53.199055+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:22:53.199Z" }, { "sequence": 2665, "attempt_id": "472eb7a6-558a-44d8-a08b-4716d17d6661", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.363664577016607, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:53.548273+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:22:53.548Z" }, { "sequence": 2666, "attempt_id": "d6371cd0-e74b-4a74-848a-5c7cf6fd9a5f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1041931869694963, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:54.002001+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:22:54.002Z" }, { "sequence": 2667, "attempt_id": "66c31067-eb95-438e-b39a-2de975193bce", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044265, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.485293519916013, "model_name": "muse-spark-1.3", "output_tokens": 298, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:54.248628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0044265, "created_at": "2026-09-17T03:22:54.248Z" }, { "sequence": 2668, "attempt_id": "c04cd81f-9121-4f6c-930e-cbe0dfc998f6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1432304210029542, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:54.389562+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:22:54.391Z" }, { "sequence": 2669, "attempt_id": "528117c5-61f7-49fe-8baa-6dc7ddfa29c4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.9788258839398623, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:54.539674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:22:54.539Z" }, { "sequence": 2670, "attempt_id": "44ee487d-8655-4ab1-9f90-b5392548f50f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005224, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6069251610897481, "model_name": "gpt-5.6-terra", "output_tokens": 37, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:54.677843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005224, "created_at": "2026-09-17T03:22:54.678Z" }, { "sequence": 2671, "attempt_id": "6b970816-8e08-4011-8142-490c7ff43591", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001329, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.1827767750946805, "model_name": "gemini-3.8-flash", "output_tokens": 127, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:54.869363+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001329, "created_at": "2026-09-17T03:22:54.869Z" }, { "sequence": 2672, "attempt_id": "cf310470-aa97-409a-bb5f-d747e28d21e8", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8605371559970081, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:55.008549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:22:55.008Z" }, { "sequence": 2673, "attempt_id": "f96bf23d-353b-4ecb-97ae-a2a4958fe9e4", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.466801682021469, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:55.647327+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:22:55.647Z" }, { "sequence": 2674, "attempt_id": "bc2682ad-cb4b-4d31-b3f4-09c63c6658f1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.7499516110401601, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:55.909842+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T03:22:55.909Z" }, { "sequence": 2675, "attempt_id": "2c5ec94d-995f-4c70-af71-79801d92215c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041625, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.7418662779964507, "model_name": "muse-spark-1.2", "output_tokens": 235, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:56.281838+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0041625, "created_at": "2026-09-17T03:22:56.281Z" }, { "sequence": 2676, "attempt_id": "826a1cb8-66a6-498a-8147-ddc67cec6d53", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.0686364819994196, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:56.885438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T03:22:56.885Z" }, { "sequence": 2677, "attempt_id": "9b9e0e39-25e2-4a67-9318-94c4fdd9ddbf", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003342, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.734972445992753, "model_name": "gemini-3.1-pro-preview", "output_tokens": 89, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:57.082689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003342, "created_at": "2026-09-17T03:22:57.082Z" }, { "sequence": 2678, "attempt_id": "6df8723c-cd20-4e4e-94a9-3f8635f21e88", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.227374981972389, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:57.236190+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T03:22:57.236Z" }, { "sequence": 2679, "attempt_id": "819aef42-7491-4085-afa7-01154da8f1d7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.2083218509797007, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:57.374414+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:22:57.374Z" }, { "sequence": 2680, "attempt_id": "e013d857-518d-4b7c-aec2-da27ec0622ec", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005032, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1263177009532228, "model_name": "gpt-5.6-terra", "output_tokens": 25, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:57.549470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005032, "created_at": "2026-09-17T03:22:57.549Z" }, { "sequence": 2681, "attempt_id": "6a33229f-da8b-4f04-80e9-74477d6fdcfb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005123, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 4.047441366943531, "model_name": "muse-spark-1.3", "output_tokens": 461, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:58.437347+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005123, "created_at": "2026-09-17T03:22:58.437Z" }, { "sequence": 2682, "attempt_id": "fdbcfde2-32f4-4029-a2b0-1eb0d93d6a2d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0622982999775559, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:58.584878+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:22:58.585Z" }, { "sequence": 2683, "attempt_id": "79f6c77e-924e-4044-aa9c-569fec8f99b4", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.957286187913269, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:58.729252+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T03:22:58.730Z" }, { "sequence": 2684, "attempt_id": "8f75fe5f-1130-4d3f-a084-1ac5c5d4535a", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00130125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.4992628738982603, "model_name": "gemini-3.8-flash", "output_tokens": 124, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:58.873968+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00130125, "created_at": "2026-09-17T03:22:58.874Z" }, { "sequence": 2685, "attempt_id": "13d610ae-68bf-499c-80c9-3f6d71a9a39c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 3.479916153009981, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:59.268546+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:22:59.268Z" }, { "sequence": 2686, "attempt_id": "d4e87590-9fa1-46b3-89d2-4463eb99bc69", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.8356554019264877, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:59.875546+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:22:59.875Z" }, { "sequence": 2687, "attempt_id": "2df233b3-5d0e-496c-8c18-133cc944dd91", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.1496509860735387, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:00.176267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:23:00.176Z" }, { "sequence": 2688, "attempt_id": "d0f78299-335b-4c97-905a-0bfd41e1fb09", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.0822979880031198, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:00.494239+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:23:00.494Z" }, { "sequence": 2689, "attempt_id": "1bc502e8-b004-487d-8671-37b7c9f76f25", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.211859790957533, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:01.083232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:23:01.083Z" }, { "sequence": 2690, "attempt_id": "06365aef-d327-4342-a013-291c84d1aa10", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005578, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.056100015994161, "model_name": "gemini-3.1-pro-preview", "output_tokens": 279, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:23:01.278536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005578, "created_at": "2026-09-17T03:23:01.278Z" }, { "sequence": 2691, "attempt_id": "f51fdd90-6367-46ba-9cbc-4620e7eedf08", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00439175, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.7992785869864747, "model_name": "muse-spark-1.2", "output_tokens": 296, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:01.421571+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00439175, "created_at": "2026-09-17T03:23:01.422Z" }, { "sequence": 2692, "attempt_id": "94cfe10e-9dc0-4655-996f-bd68be029568", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.237722622929141, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:01.562991+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:23:01.563Z" }, { "sequence": 2693, "attempt_id": "c7027b5f-aaaa-4a8f-80cf-601c7b11365f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0954881970537826, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:01.742531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:23:01.742Z" }, { "sequence": 2694, "attempt_id": "55131788-69a9-4ded-9a2f-d569c2f9cf49", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0050165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 4.0966760700102895, "model_name": "muse-spark-1.3", "output_tokens": 443, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:01.883892+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0050165, "created_at": "2026-09-17T03:23:01.884Z" }, { "sequence": 2695, "attempt_id": "1f932b7e-5164-4251-beae-5e54cf233792", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.0638372290413827, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:02.083894+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:23:02.084Z" }, { "sequence": 2696, "attempt_id": "b67cf89a-6ed3-4f26-a92c-218f0927345e", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.8196095380699262, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:02.535278+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:23:02.535Z" }, { "sequence": 2697, "attempt_id": "a7489535-d8d5-4b15-b5c8-cb541b3efd7c", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00120525, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.7218739830423146, "model_name": "gemini-3.8-flash", "output_tokens": 98, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:03.285198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00120525, "created_at": "2026-09-17T03:23:03.285Z" }, { "sequence": 2698, "attempt_id": "41659a79-ce11-428d-8ecb-f9643d5272f7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038655, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.7197849299991503, "model_name": "muse-spark-1.2", "output_tokens": 171, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:03.743796+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0038655, "created_at": "2026-09-17T03:23:03.743Z" }, { "sequence": 2699, "attempt_id": "062a99de-b100-4d09-ab1e-81f35fc8401d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02681225, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 30.552193701965734, "model_name": "muse-spark-1.2", "output_tokens": 5557, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:03.927180+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02681225, "created_at": "2026-09-17T03:23:03.927Z" }, { "sequence": 2700, "attempt_id": "41789a2a-b931-406d-8264-2babcbb62b3f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.7394414129666984, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:04.067853+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:23:04.068Z" }, { "sequence": 2701, "attempt_id": "58bb66bf-aba4-47a0-8234-960a8a9be253", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00411625, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.2714380259858444, "model_name": "muse-spark-1.3", "output_tokens": 230, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:04.221536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00411625, "created_at": "2026-09-17T03:23:04.221Z" }, { "sequence": 2702, "attempt_id": "3727a07a-c9c9-4234-a257-cf4ba66adb1d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 5.609500819933601, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:04.363542+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:23:04.363Z" }, { "sequence": 2703, "attempt_id": "cbf727b9-f5b1-451c-a361-744a432b2f2f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.6131517040776089, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:04.505266+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:23:04.505Z" }, { "sequence": 2704, "attempt_id": "83c1bc65-ed66-4e47-a369-d0b88d62ba77", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00359, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.986867479979992, "model_name": "gemini-3.1-pro-preview", "output_tokens": 113, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:04.640429+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00359, "created_at": "2026-09-17T03:23:04.640Z" }, { "sequence": 2705, "attempt_id": "64b38e36-f8d9-473c-9995-58bdb8b4273e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.2346341790398583, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:04.923669+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T03:23:04.923Z" }, { "sequence": 2706, "attempt_id": "bfd00c89-6317-4c82-8e4f-65ddaa799dc4", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9366119540063664, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:05.364654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T03:23:05.364Z" }, { "sequence": 2707, "attempt_id": "e81c65d0-9c9f-49de-98d3-8d99c6664ac3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.3198316359193996, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:05.541640+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:23:05.541Z" }, { "sequence": 2708, "attempt_id": "82290faf-4336-4512-a44f-06ce579bdea6", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.737586137955077, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:05.964236+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:23:05.964Z" }, { "sequence": 2709, "attempt_id": "5d8ad0df-831c-47db-aea1-8a439ef2bb51", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.9688667790032923, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:06.118891+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036389999999999995, "created_at": "2026-09-17T03:23:06.120Z" }, { "sequence": 2710, "attempt_id": "85026339-f705-4398-a56b-0845e90b5187", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-013", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.4368630199460313, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:23:06.942446+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:23:06.942Z" }, { "sequence": 2711, "attempt_id": "a26f928e-deb2-4683-990e-ec8bb0e9e2d9", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.014348943019286, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:07.089711+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:23:07.091Z" }, { "sequence": 2712, "attempt_id": "a3acbdb3-0eed-45b0-930e-37128503ecee", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002451, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.424420356983319, "model_name": "gemini-3.8-flash", "output_tokens": 421, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:07.234043+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002451, "created_at": "2026-09-17T03:23:07.234Z" }, { "sequence": 2713, "attempt_id": "f76d91ab-81e0-4641-91ac-9b3294d8202e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006144, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.761166985030286, "model_name": "gpt-5.6-terra", "output_tokens": 109, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:07.381717+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006144, "created_at": "2026-09-17T03:23:07.381Z" }, { "sequence": 2714, "attempt_id": "99d8f487-1ac1-4e81-8c3f-81772505095c", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6840606479672715, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:07.918412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:23:07.918Z" }, { "sequence": 2715, "attempt_id": "6fef4f94-6d29-4dfe-ad1a-d8e790786cfd", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.114649279974401, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:08.376137+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:23:08.376Z" }, { "sequence": 2716, "attempt_id": "a1e4cb73-a041-4281-a98c-c4f597921633", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6405322019709274, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:08.730551+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:23:08.730Z" }, { "sequence": 2717, "attempt_id": "fd09509e-d4c2-43f8-8c19-76ee9b7c75b3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0880251029739156, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:09.151424+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:23:09.151Z" }, { "sequence": 2718, "attempt_id": "57a16522-1bca-4f41-8bee-9b8950b66928", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.7714059749851003, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:09.296896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:23:09.297Z" }, { "sequence": 2719, "attempt_id": "80fa4836-a17b-4a31-b58b-99930969e032", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-013", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005398, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.416903281933628, "model_name": "gemini-3.1-pro-preview", "output_tokens": 256, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:23:09.434326+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005398, "created_at": "2026-09-17T03:23:09.434Z" }, { "sequence": 2720, "attempt_id": "e6e0f884-4414-494c-a59c-619688f379b5", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.1376475270371884, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:09.659767+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:23:09.659Z" }, { "sequence": 2721, "attempt_id": "ff73a315-78e0-4edc-96ee-cf0b10f131f4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.300634814077057, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:09.827050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:23:09.827Z" }, { "sequence": 2722, "attempt_id": "7dc47d8b-f65b-4653-8e4b-b332a00f59c5", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.791079943999648, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:09.970811+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:23:09.972Z" }, { "sequence": 2723, "attempt_id": "ec42858b-cf2b-4722-b086-fa34bd20da04", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00134775, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.0356479549081996, "model_name": "gemini-3.8-flash", "output_tokens": 132, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:10.332834+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00134775, "created_at": "2026-09-17T03:23:10.332Z" }, { "sequence": 2724, "attempt_id": "15e5008f-376d-4bd5-99ae-fe3b88cafaeb", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.3992469719378278, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:10.973076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:23:10.973Z" }, { "sequence": 2725, "attempt_id": "fcaa6703-215f-4fc4-ae95-6ec535acc98d", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6823796370299533, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:11.811147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:23:11.811Z" }, { "sequence": 2726, "attempt_id": "54584cf8-890a-454c-a34a-c4eb12b26675", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00437925, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.4247809859225526, "model_name": "muse-spark-1.3", "output_tokens": 286, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:12.003098+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00437925, "created_at": "2026-09-17T03:23:12.003Z" }, { "sequence": 2727, "attempt_id": "d0cb751c-99c7-4587-b231-7fa81fd620f1", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9374059529509395, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:12.147312+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:23:12.149Z" }, { "sequence": 2728, "attempt_id": "64df6964-ba69-49a7-af62-eefeec7a7464", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.6969227449735627, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:12.289844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:23:12.290Z" }, { "sequence": 2729, "attempt_id": "7579fdd5-8774-45f3-b6da-8e6b3048daea", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005802, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.006356008932926, "model_name": "gemini-3.1-pro-preview", "output_tokens": 294, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:12.892843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005802, "created_at": "2026-09-17T03:23:12.892Z" }, { "sequence": 2730, "attempt_id": "45f538ad-2c89-4cc1-b01c-ae9bf55dddc9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9616555969696492, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:13.405952+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:23:13.406Z" }, { "sequence": 2731, "attempt_id": "f7d050e9-1294-4219-a819-3afde1318dc5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1204570269910619, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:13.549844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:23:13.550Z" }, { "sequence": 2732, "attempt_id": "112c3082-b682-4666-8c9a-2a0813c3c896", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.5523621239699423, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:13.699943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:23:13.700Z" }, { "sequence": 2733, "attempt_id": "a75c4b5e-31b9-43d6-8eed-115aafe94f1b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 4.265403012977913, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:14.236503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:23:14.236Z" }, { "sequence": 2734, "attempt_id": "af0905ea-f343-49b7-8b2f-b3b979d88591", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.861596364993602, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:14.554656+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:23:14.554Z" }, { "sequence": 2735, "attempt_id": "b5d2a6b6-4b8d-45a8-b6c2-70799d58aecc", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.0560475320089608, "model_name": "gemini-3.8-flash", "output_tokens": 101, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:14.702945+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00122925, "created_at": "2026-09-17T03:23:14.703Z" }, { "sequence": 2736, "attempt_id": "98ef7064-1d39-41a1-a6f6-f96b46c5fc78", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 3.006688574911095, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:14.963677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:23:14.963Z" }, { "sequence": 2737, "attempt_id": "5164a6fb-b9dd-415e-9350-78700ed139f6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00364875, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 0.9282357629854232, "model_name": "muse-spark-1.2", "output_tokens": 115, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:15.309832+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00364875, "created_at": "2026-09-17T03:23:15.309Z" }, { "sequence": 2738, "attempt_id": "ed8620c8-0858-4f39-b38f-19e0bf1aa632", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.7187646010424942, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:16.170183+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T03:23:16.170Z" }, { "sequence": 2739, "attempt_id": "b2b562ec-3638-43d1-b193-3fa1ebebdac1", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003564, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.1591077360790223, "model_name": "gemini-3.1-pro-preview", "output_tokens": 108, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:16.318765+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003564, "created_at": "2026-09-17T03:23:16.319Z" }, { "sequence": 2740, "attempt_id": "351b0ae1-ccd8-49f7-a76d-9ff53538627c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00407375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.413697247975506, "model_name": "muse-spark-1.3", "output_tokens": 215, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:16.454062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00407375, "created_at": "2026-09-17T03:23:16.454Z" }, { "sequence": 2741, "attempt_id": "d84ae96d-2652-4ee2-9ee6-7da15c289a55", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.1388372209621593, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:16.982909+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01434, "created_at": "2026-09-17T03:23:16.983Z" }, { "sequence": 2742, "attempt_id": "028d3d84-75a4-4330-9a9d-18e7a2a04b29", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006296, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.302712721982971, "model_name": "claude-sonnet-5", "output_tokens": 67, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:17.419883+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006296, "created_at": "2026-09-17T03:23:17.420Z" }, { "sequence": 2743, "attempt_id": "42588720-8e9f-4b58-bccc-76a306ef2718", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.1098455659812316, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:17.715380+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:23:17.715Z" }, { "sequence": 2744, "attempt_id": "287d398f-49a7-46db-8ce6-c5b30768aafe", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005518, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.623903839965351, "model_name": "gpt-5.6-terra", "output_tokens": 61, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:18.233912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005518, "created_at": "2026-09-17T03:23:18.234Z" }, { "sequence": 2745, "attempt_id": "213049ff-5887-4f4d-8dc3-d274c1e2a028", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.3650759309530258, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:18.494651+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:23:18.494Z" }, { "sequence": 2746, "attempt_id": "65592adf-2526-4c41-a30f-ed3473c83085", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.9845949719892815, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:19.303682+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:23:19.303Z" }, { "sequence": 2747, "attempt_id": "2272baea-9fda-4b81-a13e-cec2cb4d0a82", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.066970085958019, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:19.456002+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:23:19.456Z" }, { "sequence": 2748, "attempt_id": "d2e22a06-f842-47ff-82d3-8a4a5a43f3c2", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00153, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 2.7840188649715856, "model_name": "gemini-3.8-flash", "output_tokens": 180, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:20.643240+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00153, "created_at": "2026-09-17T03:23:20.643Z" }, { "sequence": 2749, "attempt_id": "dee6bb8c-f83b-4645-80e8-66e2b0804ea1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02023325, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 47.48063681402709, "model_name": "muse-spark-1.3", "output_tokens": 4009, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:20.803439+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02023325, "created_at": "2026-09-17T03:23:20.803Z" }, { "sequence": 2750, "attempt_id": "4ebc6650-c67d-442e-bbfc-d03353fd85f5", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 6.0084520620293915, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:20.942510+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T03:23:20.942Z" }, { "sequence": 2751, "attempt_id": "f3903bee-cdff-4768-a64b-c64deed3b15b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01719825, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 15.259894605958834, "model_name": "muse-spark-1.2", "output_tokens": 3294, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:21.088882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01719825, "created_at": "2026-09-17T03:23:21.089Z" }, { "sequence": 2752, "attempt_id": "60412f28-f8fd-497e-87cc-722387025fc9", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00384, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.5140571910887957, "model_name": "gemini-3.1-pro-preview", "output_tokens": 130, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:21.224761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00384, "created_at": "2026-09-17T03:23:21.224Z" }, { "sequence": 2753, "attempt_id": "231099ff-7614-43cc-b5ab-9ae79a7b6106", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.5463258699746802, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:21.635490+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T03:23:21.635Z" }, { "sequence": 2754, "attempt_id": "65b0ba15-bc76-494a-9ffe-836d7c103f5f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003882, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 12.5502386019798, "model_name": "muse-spark-1.2", "output_tokens": 169, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:22.364180+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003882, "created_at": "2026-09-17T03:23:22.364Z" }, { "sequence": 2755, "attempt_id": "600fa95b-a498-49a4-8757-566f173730f5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0093956070486456, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:22.788186+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:23:22.788Z" }, { "sequence": 2756, "attempt_id": "7477ab4c-359c-492b-9537-94f4005fca00", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9363128000404686, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:23.021006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:23:23.021Z" }, { "sequence": 2757, "attempt_id": "d555633e-41d2-48bc-97c2-d0fa0f6186f5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.435085572069511, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:23.238803+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T03:23:23.238Z" }, { "sequence": 2758, "attempt_id": "c920b800-3b36-442f-addd-d907636fe2d6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.0112266129581258, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:23.390608+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:23:23.390Z" }, { "sequence": 2759, "attempt_id": "6acde56d-7ddf-4194-88da-3875c75173fe", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.237315605976619, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:23.621047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:23:23.621Z" }, { "sequence": 2760, "attempt_id": "00293b83-7712-4e5a-81a0-57f208a9aa61", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.132166153984144, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:23.765329+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T03:23:23.766Z" }, { "sequence": 2761, "attempt_id": "525b5003-a8e9-4b8d-8975-ee13977b3efe", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00516625, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 5.424577387049794, "model_name": "muse-spark-1.3", "output_tokens": 470, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:24.063943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00516625, "created_at": "2026-09-17T03:23:24.064Z" }, { "sequence": 2762, "attempt_id": "95d4454a-af6a-46d5-8121-c5f1bf871ecd", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7347398030105978, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:24.207855+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00035009999999999994, "created_at": "2026-09-17T03:23:24.208Z" }, { "sequence": 2763, "attempt_id": "b72ef7cb-7318-47b4-bcb6-6cd68afd897a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00624575, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.894386596046388, "model_name": "muse-spark-1.2", "output_tokens": 724, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:24.351804+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00624575, "created_at": "2026-09-17T03:23:24.352Z" }, { "sequence": 2764, "attempt_id": "36c6bce1-abd6-4cad-9f82-e9fe56ee5c55", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00145275, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.4497610720572993, "model_name": "gemini-3.8-flash", "output_tokens": 164, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:24.624267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00145275, "created_at": "2026-09-17T03:23:24.624Z" }, { "sequence": 2765, "attempt_id": "72b33635-b156-48c1-8c7e-e2b02cf7694b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00389525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.2191652540350333, "model_name": "muse-spark-1.2", "output_tokens": 178, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:24.984777+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00389525, "created_at": "2026-09-17T03:23:24.984Z" }, { "sequence": 2766, "attempt_id": "5f4a0c06-05ec-45a3-8f9f-ae85e211443f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6998839159496129, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:25.193720+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:23:25.193Z" }, { "sequence": 2767, "attempt_id": "ea9b27ae-3631-4aed-a3b3-df501d1d7ffb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00395475, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.7662125570932403, "model_name": "muse-spark-1.3", "output_tokens": 192, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:25.338455+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00395475, "created_at": "2026-09-17T03:23:25.338Z" }, { "sequence": 2768, "attempt_id": "86ff042e-1fdb-44d1-89e2-7a97bdd8559d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003818, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.247513381065801, "model_name": "gemini-3.1-pro-preview", "output_tokens": 132, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:26.181245+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003818, "created_at": "2026-09-17T03:23:26.181Z" }, { "sequence": 2769, "attempt_id": "2e560c2a-ecfd-41df-87a4-7fd5ea679065", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.8715016409987584, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:26.342976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:23:26.343Z" }, { "sequence": 2770, "attempt_id": "c5dcf0d0-57ee-4bd8-b303-6588f05d2650", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.9520876680035144, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:26.492195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:23:26.493Z" }, { "sequence": 2771, "attempt_id": "21ec074f-0d99-45a8-bb9f-fd43c91f9897", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0538448240840808, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:26.635707+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:23:26.635Z" }, { "sequence": 2772, "attempt_id": "a703b567-19da-4616-b909-206092fbeb06", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.014443331048824, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:26.786654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:23:26.786Z" }, { "sequence": 2773, "attempt_id": "a2af3cea-8ad5-4cc0-885f-e17ab29c6de1", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.3093039579689503, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:26.941830+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:23:26.941Z" }, { "sequence": 2774, "attempt_id": "b7cde397-a737-4992-9a2b-5228b847cafa", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7888805479742587, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:27.084877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:23:27.085Z" }, { "sequence": 2775, "attempt_id": "cf46394d-bac2-48bb-94b2-edc749376ee0", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8160021550720558, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:27.598402+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:23:27.598Z" }, { "sequence": 2776, "attempt_id": "c4c97121-fd43-4914-a080-3c692ef690a7", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.807485605007969, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:27.749664+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:23:27.749Z" }, { "sequence": 2777, "attempt_id": "c5d82593-2e66-48a8-b0eb-27e7d86648be", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040095, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.169022548943758, "model_name": "muse-spark-1.2", "output_tokens": 199, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:28.402601+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0040095, "created_at": "2026-09-17T03:23:28.402Z" }, { "sequence": 2778, "attempt_id": "ff5c7c38-1052-4d44-be1f-5a2cb65674d1", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012015, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.4198996940394863, "model_name": "gemini-3.8-flash", "output_tokens": 93, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:28.912405+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0012015, "created_at": "2026-09-17T03:23:28.912Z" }, { "sequence": 2779, "attempt_id": "bd6dd7e5-dae8-4587-89d5-d970cd092258", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003354, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.7921555089997128, "model_name": "gemini-3.1-pro-preview", "output_tokens": 90, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:29.135424+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003354, "created_at": "2026-09-17T03:23:29.135Z" }, { "sequence": 2780, "attempt_id": "3e63bf02-d764-4a2d-a154-1912745f9b28", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 1.272428939002566, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:29.279850+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T03:23:29.280Z" }, { "sequence": 2781, "attempt_id": "dc963af4-b344-48f9-9f01-48a513d28942", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.0269071869552135, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:29.432219+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T03:23:29.432Z" }, { "sequence": 2782, "attempt_id": "b62140f4-0d7f-437d-8387-42ffbf6fd0cd", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00432825, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.691048313979991, "model_name": "muse-spark-1.3", "output_tokens": 274, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:29.580431+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00432825, "created_at": "2026-09-17T03:23:29.580Z" }, { "sequence": 2783, "attempt_id": "63a2af44-75a4-433a-b6bf-cda5cb7bd198", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.9383013639599085, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:29.726013+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T03:23:29.726Z" }, { "sequence": 2784, "attempt_id": "c6cee646-2919-4ad2-9c29-fdd5f77ac3e9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 4.9313152260147035, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:30.068088+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:23:30.068Z" }, { "sequence": 2785, "attempt_id": "71d80a9d-b8ec-4042-949f-8832b584f7d3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.013981, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 24.60176968004089, "model_name": "muse-spark-1.3", "output_tokens": 2537, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:30.227801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.013981, "created_at": "2026-09-17T03:23:30.228Z" }, { "sequence": 2786, "attempt_id": "952894dd-5c71-4148-a5ca-4fab4c0d9ce5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.0808280620258301, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:30.374929+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:23:30.375Z" }, { "sequence": 2787, "attempt_id": "f75d345f-3534-480a-b667-883882dd6e44", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.4540060999570414, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:30.518366+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:23:30.518Z" }, { "sequence": 2788, "attempt_id": "d0cb7b5d-ccc3-4ec8-90b9-0e7d0b31c907", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.072408748907037, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:30.680866+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:23:30.681Z" }, { "sequence": 2789, "attempt_id": "278f4bb5-b93a-4975-adda-2526ef3308c9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003495, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.8556215089047328, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:30.826955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003495, "created_at": "2026-09-17T03:23:30.827Z" }, { "sequence": 2790, "attempt_id": "d8ff4e14-08fd-4bbe-8968-538ffa8e2c27", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.1659054300980642, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:30.967254+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:23:30.967Z" }, { "sequence": 2791, "attempt_id": "18b3d565-5af5-4275-9821-52d3d161bce1", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 4.572299193008803, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:31.810225+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T03:23:31.810Z" }, { "sequence": 2792, "attempt_id": "5f342bae-b918-48d2-ba4d-8cfd2086653e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.4364301969762892, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:32.263730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:23:32.263Z" }, { "sequence": 2793, "attempt_id": "ed39af58-e616-4c44-85c1-bbe8b1c8cb49", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.1921448420034721, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:32.414291+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:23:32.414Z" }, { "sequence": 2794, "attempt_id": "f9f6d954-2658-484a-8e4e-f40332d98022", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00118125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.5750894400989637, "model_name": "gemini-3.8-flash", "output_tokens": 92, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:32.553740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00118125, "created_at": "2026-09-17T03:23:32.553Z" }, { "sequence": 2795, "attempt_id": "48f28e47-7b6f-4577-b8ba-182993a3a74f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004991, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.4019865830196068, "model_name": "muse-spark-1.2", "output_tokens": 437, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:32.777194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004991, "created_at": "2026-09-17T03:23:32.777Z" }, { "sequence": 2796, "attempt_id": "07e0eb15-2d10-463b-b086-4795b35a5c5b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.8976690049748868, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:33.012010+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:23:33.012Z" }, { "sequence": 2797, "attempt_id": "0c43da40-1ab6-4fb0-b9b2-379f1060e8ee", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.277066086884588, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:33.248110+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:23:33.248Z" }, { "sequence": 2798, "attempt_id": "c0b0d078-ffb4-48dc-a461-22b86e8fb6bc", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.5903819140512496, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:33.393104+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T03:23:33.393Z" }, { "sequence": 2799, "attempt_id": "87630261-bd8a-4564-9d31-012ffae21a98", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00508, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0438787839375436, "model_name": "gpt-5.6-terra", "output_tokens": 25, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:33.538334+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00508, "created_at": "2026-09-17T03:23:33.538Z" }, { "sequence": 2800, "attempt_id": "df624e41-894c-4f64-91c0-d70d31b7a61b", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005578, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.195109204039909, "model_name": "gemini-3.1-pro-preview", "output_tokens": 279, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:33.775808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005578, "created_at": "2026-09-17T03:23:33.775Z" }, { "sequence": 2801, "attempt_id": "1825c861-be43-4a53-aeb4-835545f6188c", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8049801250454038, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:33.961417+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003561, "created_at": "2026-09-17T03:23:33.961Z" }, { "sequence": 2802, "attempt_id": "2dde33ba-47c7-48ac-9214-1bcf5ae4faba", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012915, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.100816699094139, "model_name": "gemini-3.8-flash", "output_tokens": 117, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:23:34.106516+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0012915, "created_at": "2026-09-17T03:23:34.106Z" }, { "sequence": 2803, "attempt_id": "495cb2a3-6926-4097-8f8d-8b3b20a672d3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.270972686004825, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:34.244975+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:23:34.245Z" }, { "sequence": 2804, "attempt_id": "40cf3280-daa1-4d76-8c30-4a121340835a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.8611769980052486, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:34.408194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T03:23:34.408Z" }, { "sequence": 2805, "attempt_id": "b4401531-a7d8-4747-ab0f-a14d4d14ebd0", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004923, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 4.02433046803344, "model_name": "muse-spark-1.3", "output_tokens": 421, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:34.550898+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004923, "created_at": "2026-09-17T03:23:34.551Z" }, { "sequence": 2806, "attempt_id": "c9ba44b2-b1b4-4417-bf1e-6dee434bbac8", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.7162115890532732, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:35.120014+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T03:23:35.120Z" }, { "sequence": 2807, "attempt_id": "09ee20c5-3cfc-46db-a31a-338d5764e1c9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00386925, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.689811595948413, "model_name": "muse-spark-1.2", "output_tokens": 166, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:35.269954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00386925, "created_at": "2026-09-17T03:23:35.270Z" }, { "sequence": 2808, "attempt_id": "1dd13dc8-5c8e-4572-a77e-20ca64343f7d", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.2724044480128214, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:35.428843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00571, "created_at": "2026-09-17T03:23:35.430Z" }, { "sequence": 2809, "attempt_id": "26dfb02c-781f-4705-b7fb-a3159478ab26", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003882, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.9509597410215065, "model_name": "muse-spark-1.3", "output_tokens": 169, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:35.568257+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003882, "created_at": "2026-09-17T03:23:35.568Z" }, { "sequence": 2810, "attempt_id": "d156bb2e-e760-4727-b98a-d71b46cb42ad", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005096, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.197528864024207, "model_name": "gpt-5.6-terra", "output_tokens": 28, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:35.896536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005096, "created_at": "2026-09-17T03:23:35.896Z" }, { "sequence": 2811, "attempt_id": "ad9d1b88-814e-4be3-9a8e-65a053077ef9", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.1485062709543854, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:36.070464+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014275, "created_at": "2026-09-17T03:23:36.070Z" }, { "sequence": 2812, "attempt_id": "d8af7a41-a071-43b3-a6c1-a039d49edfc3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.4327031419379637, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:36.227815+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:23:36.228Z" }, { "sequence": 2813, "attempt_id": "17126440-3fba-4607-b5f9-88af648130bd", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005358, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.913284022011794, "model_name": "gemini-3.1-pro-preview", "output_tokens": 257, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:23:36.614811+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005358, "created_at": "2026-09-17T03:23:36.614Z" }, { "sequence": 2814, "attempt_id": "b69334d3-c0b7-4467-b408-c5fb1caafb99", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.0666452830191702, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:36.782791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:23:36.782Z" }, { "sequence": 2815, "attempt_id": "22c00225-b6dc-473f-ab95-9c61409e8b31", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00150525, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.3144246429437771, "model_name": "gemini-3.8-flash", "output_tokens": 176, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:37.026134+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00150525, "created_at": "2026-09-17T03:23:37.026Z" }, { "sequence": 2816, "attempt_id": "cfaa5e01-854a-4527-8d72-2c834b6b67cc", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.6781330050434917, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:37.187091+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:23:37.187Z" }, { "sequence": 2817, "attempt_id": "4afc03e0-d916-45c6-a9ec-3acba2e21e47", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.34079084394034, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:37.331097+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02857, "created_at": "2026-09-17T03:23:37.331Z" }, { "sequence": 2818, "attempt_id": "c0296adb-2f04-4005-bb8e-ffab02cbb29e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00414275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.590652244980447, "model_name": "muse-spark-1.2", "output_tokens": 233, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:37.818779+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00414275, "created_at": "2026-09-17T03:23:37.818Z" }, { "sequence": 2819, "attempt_id": "5b5611c9-cdb2-428b-ae4e-a693780a50be", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-017", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001928, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7947205269010738, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "C" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:37.982096+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001928, "created_at": "2026-09-17T03:23:37.982Z" }, { "sequence": 2820, "attempt_id": "383c89af-445e-4a34-8e01-bd00f529cdd5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.75395717297215, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:38.139788+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:23:38.140Z" }, { "sequence": 2821, "attempt_id": "aaa88c97-a41c-4602-9c02-b401d917a354", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004215, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 2.397384680924006, "model_name": "muse-spark-1.3", "output_tokens": 250, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:38.442212+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004215, "created_at": "2026-09-17T03:23:38.442Z" }, { "sequence": 2822, "attempt_id": "8870b843-ea77-4902-8287-81feb3052451", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005544, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.6834599430439994, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:39.651109+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005544, "created_at": "2026-09-17T03:23:39.651Z" }, { "sequence": 2823, "attempt_id": "fbf8b8a8-1091-4918-88df-76394c3e0a8f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006226, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.343043891945854, "model_name": "gemini-3.1-pro-preview", "output_tokens": 331, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:39.803004+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006226, "created_at": "2026-09-17T03:23:39.803Z" }, { "sequence": 2824, "attempt_id": "6c29c2bd-5389-4ef2-bab3-78b870368d38", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011404, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.3151222029700875, "model_name": "gpt-5.6-sol", "output_tokens": 77, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:39.957938+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011404, "created_at": "2026-09-17T03:23:39.958Z" }, { "sequence": 2825, "attempt_id": "c4d798ab-e156-4961-8a99-ca9435002d76", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005952, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7491669830633327, "model_name": "gpt-5.6-luna", "output_tokens": 85, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:40.107725+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005952, "created_at": "2026-09-17T03:23:40.107Z" }, { "sequence": 2826, "attempt_id": "5e6b79fc-72b9-4575-a331-f64016aa808f", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02721, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.5115085219731554, "model_name": "gpt-6-astra", "output_tokens": 51, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:40.249448+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02721, "created_at": "2026-09-17T03:23:40.249Z" }, { "sequence": 2827, "attempt_id": "7138c0f2-cfd1-47e3-844d-fadc0f5fb037", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018045, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.623408325947821, "model_name": "claude-opus-5", "output_tokens": 135, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:40.397961+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.018045, "created_at": "2026-09-17T03:23:40.399Z" }, { "sequence": 2828, "attempt_id": "6837a1b2-a41c-49fb-a8ef-ba668caeb284", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-017", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.6595720970071852, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:23:40.556399+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:23:40.556Z" }, { "sequence": 2829, "attempt_id": "f46e941c-c806-4c91-9680-eb227df52b3c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008038, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.652394859935157, "model_name": "claude-sonnet-5", "output_tokens": 217, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:23:40.702822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008038, "created_at": "2026-09-17T03:23:40.703Z" }, { "sequence": 2830, "attempt_id": "d1de3eb0-3a1e-4c44-9c6e-455cb9f0f1d2", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.5499372149351984, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:41.253056+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001876, "created_at": "2026-09-17T03:23:41.253Z" }, { "sequence": 2831, "attempt_id": "347db217-9819-4351-91d5-eaf5c6101d97", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00312225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.8676504539325833, "model_name": "gemini-3.8-flash", "output_tokens": 590, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:23:41.457202+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00312225, "created_at": "2026-09-17T03:23:41.457Z" }, { "sequence": 2832, "attempt_id": "6605725e-a96f-4675-83fb-40c7f3167463", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03041, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 5.444923617062159, "model_name": "claude-fable-5-1", "output_tokens": 21, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:23:41.817937+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03041, "created_at": "2026-09-17T03:23:41.818Z" }, { "sequence": 2833, "attempt_id": "23832333-2e1a-42a4-92e6-1432e7a08fdf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005683999999999999, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.439207509974949, "model_name": "gpt-5.6-luna", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:43.406200+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005683999999999999, "created_at": "2026-09-17T03:23:43.406Z" }, { "sequence": 2834, "attempt_id": "7b0b7aea-30a8-4b6b-9a83-6fcf16b4c9c2", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02674, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.7181236629839987, "model_name": "gpt-6-astra", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:43.568841+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02674, "created_at": "2026-09-17T03:23:43.568Z" }, { "sequence": 2835, "attempt_id": "ef360fac-9ef9-4a0d-95d7-ce0f43a86c4c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005924, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.088609583908692, "model_name": "gpt-5.6-terra", "output_tokens": 93, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:43.730060+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005924, "created_at": "2026-09-17T03:23:43.731Z" }, { "sequence": 2836, "attempt_id": "d27c08b2-df66-49f2-bc93-077680d2dd95", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007788, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.0621805479750037, "model_name": "claude-sonnet-5", "output_tokens": 210, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:43.871731+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007788, "created_at": "2026-09-17T03:23:43.871Z" }, { "sequence": 2837, "attempt_id": "9582bbfc-fdff-4e24-886a-cf937707ad69", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011836, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 3.112789222970605, "model_name": "gpt-5.6-sol", "output_tokens": 111, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:44.528968+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011836, "created_at": "2026-09-17T03:23:44.529Z" }, { "sequence": 2838, "attempt_id": "91a8a52b-7583-4b7b-afd9-f01d436aeb98", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.7820789720863104, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:44.800764+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000374, "created_at": "2026-09-17T03:23:44.800Z" }, { "sequence": 2839, "attempt_id": "06b312d7-d5da-4224-8901-bbdf65a453d3", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017595, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 5.011828515911475, "model_name": "claude-opus-5", "output_tokens": 135, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:45.410077+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017595, "created_at": "2026-09-17T03:23:45.410Z" }, { "sequence": 2840, "attempt_id": "14571904-0317-41a3-8b38-0b7a62174760", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03711, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 5.84420070098713, "model_name": "claude-fable-5-1", "output_tokens": 173, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:46.237830+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03711, "created_at": "2026-09-17T03:23:46.238Z" }, { "sequence": 2841, "attempt_id": "2859d394-fffc-4de7-9fe1-3b55e182b14e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0033975, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 4.22948754299432, "model_name": "gemini-3.8-flash", "output_tokens": 675, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:47.959876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0033975, "created_at": "2026-09-17T03:23:47.960Z" }, { "sequence": 2842, "attempt_id": "4e024308-9246-4384-8f03-629c828538b7", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.95495464396663, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:48.343616+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T03:23:48.343Z" }, { "sequence": 2843, "attempt_id": "f93fd3a7-e47d-42f2-9cc2-9c6495ff3203", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016526, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 10.486833870992996, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1175, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "google", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:23:48.772085+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016526, "created_at": "2026-09-17T03:23:48.772Z" }, { "sequence": 2844, "attempt_id": "9f87a612-55e7-4860-8ca0-49a2766abd8b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.895132091944106, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:48.922794+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T03:23:48.923Z" }, { "sequence": 2845, "attempt_id": "6e92cc65-935e-425b-9a23-3840b88a2b11", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.335263382992707, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:49.068474+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T03:23:49.068Z" }, { "sequence": 2846, "attempt_id": "bad0eae4-d72b-4af1-a03a-9bf6fce94823", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.8963161070132628, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:23:49.218944+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T03:23:49.221Z" }, { "sequence": 2847, "attempt_id": "a482c052-244e-4099-b17b-f13d8ead5578", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.7790880149696022, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:50.285899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:23:50.286Z" }, { "sequence": 2848, "attempt_id": "a4d1cc34-7b7c-416c-85fd-eb0adc464a1e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010512, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.5463072420097888, "model_name": "gpt-5.6-sol", "output_tokens": 42, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:50.469381+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010512, "created_at": "2026-09-17T03:23:50.469Z" }, { "sequence": 2849, "attempt_id": "9e7b9e6c-7a53-4e99-9bd1-e28cfe9b4d35", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4408437630627304, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:50.660081+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:23:50.660Z" }, { "sequence": 2850, "attempt_id": "8bf69654-eaa4-446d-b854-065deca7592c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01185225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.973216101992875, "model_name": "muse-spark-1.2", "output_tokens": 2022, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:23:51.081230+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01185225, "created_at": "2026-09-17T03:23:51.081Z" }, { "sequence": 2851, "attempt_id": "bd5c0364-5bdd-4109-ac6b-4744de05a995", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00602025, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 7.2748046070337296, "model_name": "muse-spark-1.3", "output_tokens": 668, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:51.298113+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00602025, "created_at": "2026-09-17T03:23:51.298Z" }, { "sequence": 2852, "attempt_id": "c24220fe-adc7-4acc-b04d-1e924eda4e96", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.901230383082293, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:51.516835+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036389999999999995, "created_at": "2026-09-17T03:23:51.516Z" }, { "sequence": 2853, "attempt_id": "1ace6795-bd4d-40d5-80aa-227526fc7059", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.0404165079817176, "model_name": "gpt-5.6-terra", "output_tokens": 93, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:52.254821+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005952, "created_at": "2026-09-17T03:23:52.255Z" }, { "sequence": 2854, "attempt_id": "10d81f63-c433-4c45-b0bc-968e19dae729", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002871, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.48307457996998, "model_name": "gemini-3.8-flash", "output_tokens": 533, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:23:52.920244+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002871, "created_at": "2026-09-17T03:23:52.920Z" }, { "sequence": 2855, "attempt_id": "fbe2a2af-eb22-40ba-8e81-3da879691a03", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077765, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 13.915445372927934, "model_name": "muse-spark-1.3", "output_tokens": 1063, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:54.018839+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0077765, "created_at": "2026-09-17T03:23:54.019Z" }, { "sequence": 2856, "attempt_id": "f663efdf-c8e4-4d73-8f2c-0762dd4c9283", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.9711415260098875, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:54.172198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T03:23:54.172Z" }, { "sequence": 2857, "attempt_id": "16aa0b8d-1c0a-4f4c-ae4a-99b28618b3a5", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.685533793992363, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:54.315286+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T03:23:54.315Z" }, { "sequence": 2858, "attempt_id": "73f20758-aaa2-4e59-a9d0-fbc5f516be39", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.648924603010528, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:54.466158+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T03:23:54.466Z" }, { "sequence": 2859, "attempt_id": "7ab32182-15fd-4b28-93c3-6d326ab936f9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0404409440234303, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:55.506903+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:23:55.507Z" }, { "sequence": 2860, "attempt_id": "53276002-7a71-4bd4-87fa-a4e386ae2d11", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9940290009835735, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:55.660552+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:23:55.660Z" }, { "sequence": 2861, "attempt_id": "c307f21f-8255-4f61-8290-586cb3720434", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9021673829993233, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:56.074669+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:23:56.074Z" }, { "sequence": 2862, "attempt_id": "b078b28f-4e38-429f-bb46-3d2d071d32e3", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 5.154253747081384, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:56.599283+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T03:23:56.599Z" }, { "sequence": 2863, "attempt_id": "df8f56d7-e388-4204-9562-90dac8070bea", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.2729088639607653, "model_name": "gemini-3.8-flash", "output_tokens": 119, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:57.082112+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00129675, "created_at": "2026-09-17T03:23:57.082Z" }, { "sequence": 2864, "attempt_id": "6a83bf0b-ce81-4854-aa8f-8ecccf76d410", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.740166440955363, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:57.234199+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:23:57.235Z" }, { "sequence": 2865, "attempt_id": "247fc307-10d0-4b41-ade3-eb2fbdd953fd", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003552, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.8632639140123501, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:57.377141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003552, "created_at": "2026-09-17T03:23:57.377Z" }, { "sequence": 2866, "attempt_id": "85ef6f51-7a2d-4d99-9ee0-f6067726f25c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012442, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 8.2357761770254, "model_name": "gemini-3.1-pro-preview", "output_tokens": 843, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:57.615946+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012442, "created_at": "2026-09-17T03:23:57.616Z" }, { "sequence": 2867, "attempt_id": "edbdb084-65f7-400e-bd01-fe5ff31ef3bb", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-009", "result": { "correct": false, "cost_estimated": true, "cost_usd": 0.2548, "err": null, "error": "HTTP 400: Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.", "error_kind": "credits", "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": null, "key_errors": null, "latency_sec": 0.23329441295936704, "model_name": "claude-fable-5-1", "output_tokens": null, "prediction": {}, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.\"},\"request_id\":\"req_011Cf8J7Pcajgd7fXZr6d4b6\"}", "recorded_at": "2026-09-17T03:23:57.778871+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 1, "valid": false, "within_bands": null }, "cost_usd": 0.2548, "created_at": "2026-09-17T03:23:57.779Z" }, { "sequence": 2868, "attempt_id": "dddd5594-8980-46a9-83ce-bcbb764cefd0", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-009", "result": { "correct": false, "cost_estimated": true, "cost_usd": 0.1274, "err": null, "error": "HTTP 400: Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.", "error_kind": "credits", "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": null, "key_errors": null, "latency_sec": 0.23851612897124141, "model_name": "claude-opus-5", "output_tokens": null, "prediction": {}, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.\"},\"request_id\":\"req_011Cf8J7PbbKsRSkRSdPDc1R\"}", "recorded_at": "2026-09-17T03:23:57.925570+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 1, "valid": false, "within_bands": null }, "cost_usd": 0.1274, "created_at": "2026-09-17T03:23:57.925Z" }, { "sequence": 2869, "attempt_id": "683ac621-b260-4127-8d5d-7cedfe0f7bb4", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-009", "result": { "correct": false, "cost_estimated": true, "cost_usd": 0.05096, "err": null, "error": "HTTP 400: Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.", "error_kind": "credits", "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": null, "key_errors": null, "latency_sec": 0.14563807903323323, "model_name": "claude-sonnet-5", "output_tokens": null, "prediction": {}, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.\"},\"request_id\":\"req_011Cf8J7QfLWQjKcA9xp8km2\"}", "recorded_at": "2026-09-17T03:23:58.080536+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 1, "valid": false, "within_bands": null }, "cost_usd": 0.05096, "created_at": "2026-09-17T03:23:58.080Z" }, { "sequence": 2870, "attempt_id": "c2264508-2201-4b2c-bf72-9def3729c03d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003516, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.76162571599707, "model_name": "gemini-3.1-pro-preview", "output_tokens": 104, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:58.422495+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003516, "created_at": "2026-09-17T03:23:58.422Z" }, { "sequence": 2871, "attempt_id": "869a7236-9b2a-4a42-8a00-1cb70a786000", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00430325, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.9265115570742637, "model_name": "muse-spark-1.2", "output_tokens": 269, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:59.161020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00430325, "created_at": "2026-09-17T03:23:59.161Z" }, { "sequence": 2872, "attempt_id": "d5d0078e-244b-437a-86b1-00c1c15a843b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 0.7966333150397986, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:59.384197+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004976000000000001, "created_at": "2026-09-17T03:23:59.384Z" }, { "sequence": 2873, "attempt_id": "1d08ba4a-b886-4044-9f7f-e1972fb7b273", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01002425, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 8.186191237997264, "model_name": "muse-spark-1.2", "output_tokens": 1606, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:59.534839+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01002425, "created_at": "2026-09-17T03:23:59.535Z" }, { "sequence": 2874, "attempt_id": "5aa9ac83-df39-4833-b4b5-cb7e1efa898e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010079, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 15.655934456037357, "model_name": "muse-spark-1.2", "output_tokens": 1623, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:00.335299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010079, "created_at": "2026-09-17T03:24:00.335Z" }, { "sequence": 2875, "attempt_id": "e57c3220-7ef2-432f-ace0-1c71ef476d31", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.8638363090576604, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:00.547997+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000374, "created_at": "2026-09-17T03:24:00.548Z" }, { "sequence": 2876, "attempt_id": "12a9ed0a-347d-4bc9-934c-4dc0fefa29c7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00442225, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.979972489993088, "model_name": "muse-spark-1.3", "output_tokens": 297, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:00.739584+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00442225, "created_at": "2026-09-17T03:24:00.739Z" }, { "sequence": 2877, "attempt_id": "92d6e9c8-43f1-464c-89fa-8897f6a4d0a0", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02639, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.8057968760840595, "model_name": "gpt-6-astra", "output_tokens": 47, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:00.889950+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02639, "created_at": "2026-09-17T03:24:00.890Z" }, { "sequence": 2878, "attempt_id": "4927f17b-f2c6-4f08-a293-de15c349a274", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011116, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.836684979032725, "model_name": "gpt-5.6-sol", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:01.053452+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011116, "created_at": "2026-09-17T03:24:01.053Z" }, { "sequence": 2879, "attempt_id": "676d6b80-a6b9-4d34-b314-dc271b9d1ce0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 0.9894432399887592, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:02.193139+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004928, "created_at": "2026-09-17T03:24:02.193Z" }, { "sequence": 2880, "attempt_id": "229b23f3-5e32-413a-9ea4-ceefefa80a16", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00325875, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 2.959285240038298, "model_name": "gemini-3.8-flash", "output_tokens": 638, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:02.494422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00325875, "created_at": "2026-09-17T03:24:02.494Z" }, { "sequence": 2881, "attempt_id": "ee36f1fc-0cff-44ea-912d-cea82740976e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.366172331967391, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:03.256440+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:24:03.256Z" }, { "sequence": 2882, "attempt_id": "7c2ac5ab-9a94-402d-aa6c-e6663a8c044a", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03207, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 19.9317276700167, "model_name": "gemini-3.1-pro-preview", "output_tokens": 2480, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:03.488114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03207, "created_at": "2026-09-17T03:24:03.488Z" }, { "sequence": 2883, "attempt_id": "4efe81a4-c8f8-4fac-b5f0-c1b1e933bbcd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.4929338629590347, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:03.639161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:24:03.639Z" }, { "sequence": 2884, "attempt_id": "0bd30c40-b3bb-41db-b501-99bcebfe0ef1", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8450173130258918, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:04.484535+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:24:04.484Z" }, { "sequence": 2885, "attempt_id": "64bb0746-b36c-4804-8a1b-8c1f9a27085e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005648, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 6.537707163952291, "model_name": "gpt-5.6-terra", "output_tokens": 70, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:04.768875+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005648, "created_at": "2026-09-17T03:24:04.769Z" }, { "sequence": 2886, "attempt_id": "a4776c21-581a-42ad-bb9b-9549e19915fe", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.676359510049224, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:05.022232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:24:05.022Z" }, { "sequence": 2887, "attempt_id": "a794400b-68a5-49e4-afa8-2cd2baa9ced0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.939226842019707, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:06.110597+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:24:06.110Z" }, { "sequence": 2888, "attempt_id": "1f146d7b-8a77-4de2-b0f4-fd975e8ec08b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003977, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.6392574709607288, "model_name": "muse-spark-1.2", "output_tokens": 194, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:06.276595+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003977, "created_at": "2026-09-17T03:24:06.276Z" }, { "sequence": 2889, "attempt_id": "f8777371-949d-4074-9fb8-5ed86e7bb7bc", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005278, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 3.7549699100200087, "model_name": "gemini-3.1-pro-preview", "output_tokens": 252, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:06.425877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005278, "created_at": "2026-09-17T03:24:06.426Z" }, { "sequence": 2890, "attempt_id": "a881695f-2274-4b6f-8514-61c0247b85fe", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0014865, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 3.1711733939591795, "model_name": "gemini-3.8-flash", "output_tokens": 171, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:06.594084+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0014865, "created_at": "2026-09-17T03:24:06.594Z" }, { "sequence": 2891, "attempt_id": "14241b64-7707-4c84-aef9-bccc2b8a13d1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004521, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 3.071595216053538, "model_name": "muse-spark-1.3", "output_tokens": 322, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:06.860123+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004521, "created_at": "2026-09-17T03:24:06.860Z" }, { "sequence": 2892, "attempt_id": "d930f327-b476-4553-8909-3b5188badbd7", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.123223770991899, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:07.055947+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:24:07.056Z" }, { "sequence": 2893, "attempt_id": "738554e1-9ec9-46ce-b482-7cc86ceae6a6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9742976019624621, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:07.400463+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:24:07.400Z" }, { "sequence": 2894, "attempt_id": "06aa6916-2f1e-44ef-82eb-926d658eb686", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1979460819857195, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:07.568174+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:24:07.568Z" }, { "sequence": 2895, "attempt_id": "8e6ac51f-135c-4570-948c-3a58a50912aa", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011922, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 8.227008865913376, "model_name": "gemini-3.1-pro-preview", "output_tokens": 801, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:07.713969+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011922, "created_at": "2026-09-17T03:24:07.714Z" }, { "sequence": 2896, "attempt_id": "95ec2330-7b2c-4411-8f54-f5ce2172cd68", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7014452690491453, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:07.868963+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:24:07.869Z" }, { "sequence": 2897, "attempt_id": "9e65c5b0-61cc-4a29-8f48-1e83ba6b3a58", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00119925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.5502390449400991, "model_name": "gemini-3.8-flash", "output_tokens": 93, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:08.292931+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00119925, "created_at": "2026-09-17T03:24:08.293Z" }, { "sequence": 2898, "attempt_id": "4048847a-084f-4a06-94f1-b17e48aafe55", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00411625, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6815197770483792, "model_name": "muse-spark-1.2", "output_tokens": 225, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:09.250039+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00411625, "created_at": "2026-09-17T03:24:09.250Z" }, { "sequence": 2899, "attempt_id": "55d806f1-a198-4d57-a7af-d564d48c4a26", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9511874349555001, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:09.407193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:24:09.407Z" }, { "sequence": 2900, "attempt_id": "a6e755ad-d11e-45cc-a296-c564405a0c7f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.6588401969056576, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:09.560039+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:24:09.560Z" }, { "sequence": 2901, "attempt_id": "a84697ba-fbcf-45bd-b5fd-37cdf175f6d4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003408, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.935837685945444, "model_name": "gemini-3.1-pro-preview", "output_tokens": 95, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:09.707056+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003408, "created_at": "2026-09-17T03:24:09.707Z" }, { "sequence": 2902, "attempt_id": "3986a16a-0f59-44ee-af80-ce9f446d76a0", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.8677607310237363, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:09.875334+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:24:09.875Z" }, { "sequence": 2903, "attempt_id": "c75f7210-b8d5-4e0b-9ca9-bb24c549136e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041545, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.4012752659618855, "model_name": "muse-spark-1.3", "output_tokens": 234, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:10.021119+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0041545, "created_at": "2026-09-17T03:24:10.021Z" }, { "sequence": 2904, "attempt_id": "482e74ee-fdb1-40bf-8e39-b46701584844", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00504, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.5448733889497817, "model_name": "gpt-5.6-terra", "output_tokens": 25, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:10.163386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00504, "created_at": "2026-09-17T03:24:10.163Z" }, { "sequence": 2905, "attempt_id": "8c839085-7836-4448-b040-e549558b5a92", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7265717280097306, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:10.597445+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:24:10.597Z" }, { "sequence": 2906, "attempt_id": "ab4d46b1-8750-4582-af5e-9344feede5a6", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125775, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.2099881729809567, "model_name": "gemini-3.8-flash", "output_tokens": 112, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:10.770292+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00125775, "created_at": "2026-09-17T03:24:10.770Z" }, { "sequence": 2907, "attempt_id": "51c04792-809b-4d19-9170-c2516158a2f2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.011507, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 10.642080577090383, "model_name": "muse-spark-1.2", "output_tokens": 1959, "prediction": { "choice": "C" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:11.338063+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011507, "created_at": "2026-09-17T03:24:11.338Z" }, { "sequence": 2908, "attempt_id": "1b2e6d9d-8d25-497e-96dd-d15a3ddde346", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01180925, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 21.042858717031777, "model_name": "muse-spark-1.3", "output_tokens": 2026, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:11.850117+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01180925, "created_at": "2026-09-17T03:24:11.850Z" }, { "sequence": 2909, "attempt_id": "189bafe8-e3cc-4dd2-8b77-a087a1b062e6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007699, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 11.490154108032584, "model_name": "muse-spark-1.3", "output_tokens": 1063, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:12.004098+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007699, "created_at": "2026-09-17T03:24:12.005Z" }, { "sequence": 2910, "attempt_id": "2f345472-4eda-4024-aac8-5e35daaa5d89", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.8075546741019934, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:12.169753+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02474, "created_at": "2026-09-17T03:24:12.169Z" }, { "sequence": 2911, "attempt_id": "c5c8fabd-5217-4868-b489-54cb61e62138", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.2258430760120973, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:12.316121+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004976000000000001, "created_at": "2026-09-17T03:24:12.316Z" }, { "sequence": 2912, "attempt_id": "aa7cb16b-741c-4961-b3ea-ab386d18c099", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005624, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.5527777209645137, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:12.471874+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005624, "created_at": "2026-09-17T03:24:12.472Z" }, { "sequence": 2913, "attempt_id": "1ce5433b-12ea-46cb-b5fa-d54544c2d727", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010896, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.105701333959587, "model_name": "gpt-5.6-sol", "output_tokens": 64, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:12.621962+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010896, "created_at": "2026-09-17T03:24:12.622Z" }, { "sequence": 2914, "attempt_id": "77e5a27d-d03d-432c-8461-df4c91203ad8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00370825, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.994903240003623, "model_name": "muse-spark-1.2", "output_tokens": 134, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:12.778705+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00370825, "created_at": "2026-09-17T03:24:12.778Z" }, { "sequence": 2915, "attempt_id": "2e924907-b1ed-485e-82f0-c1eaaef92849", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004265, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.673811913933605, "model_name": "muse-spark-1.3", "output_tokens": 265, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:12.938429+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004265, "created_at": "2026-09-17T03:24:12.939Z" }, { "sequence": 2916, "attempt_id": "fa902c21-b80d-4f02-bfe4-5fd4a778ebb3", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003434, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.1919428430264816, "model_name": "gemini-3.1-pro-preview", "output_tokens": 100, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:13.082624+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003434, "created_at": "2026-09-17T03:24:13.082Z" }, { "sequence": 2917, "attempt_id": "3b669e49-e6c8-4277-94b6-11b7e98b2c36", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.4310491259675473, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:13.601101+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000374, "created_at": "2026-09-17T03:24:13.601Z" }, { "sequence": 2918, "attempt_id": "840fc709-e808-4a3a-a050-32a1a1c0d9f9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9092168110655621, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:13.847919+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:24:13.848Z" }, { "sequence": 2919, "attempt_id": "0d4f942d-a3d4-4000-8d29-54b52fe263aa", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0900103190215304, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:14.023889+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:24:14.024Z" }, { "sequence": 2920, "attempt_id": "101e8e1a-ec86-4909-b006-a1879064e18e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.8520850980421528, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:14.177016+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:24:14.177Z" }, { "sequence": 2921, "attempt_id": "721a4ca7-5b8b-47b3-9a68-f32afdab065c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.5876195400487632, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:14.516861+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:24:14.516Z" }, { "sequence": 2922, "attempt_id": "6993ba16-ccdb-443d-85c1-877dac07e6c4", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7085910870227963, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:14.710793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00035009999999999994, "created_at": "2026-09-17T03:24:14.710Z" }, { "sequence": 2923, "attempt_id": "bb21c46c-2498-4e7c-ade9-96a9be1ff592", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0011565, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.1346153180347756, "model_name": "gemini-3.8-flash", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:14.900961+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0011565, "created_at": "2026-09-17T03:24:14.901Z" }, { "sequence": 2924, "attempt_id": "d47cb193-c005-4f22-9993-c1d88c876d04", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00295125, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 2.982212887960486, "model_name": "gemini-3.8-flash", "output_tokens": 556, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:15.053536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00295125, "created_at": "2026-09-17T03:24:15.054Z" }, { "sequence": 2925, "attempt_id": "07498aab-c86b-49f1-a5b2-34b7b5cbdf47", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003806, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.0448581280652434, "model_name": "muse-spark-1.2", "output_tokens": 157, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:15.387423+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003806, "created_at": "2026-09-17T03:24:15.387Z" }, { "sequence": 2926, "attempt_id": "feda06e7-376a-4838-8119-dca1c89925fd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.8895959199871868, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:16.750782+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:24:16.750Z" }, { "sequence": 2927, "attempt_id": "d1f86bc5-4785-4e6b-8b96-71ab59bfc8a0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00561, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.758031288976781, "model_name": "gpt-5.6-terra", "output_tokens": 57, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:16.907509+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00561, "created_at": "2026-09-17T03:24:16.907Z" }, { "sequence": 2928, "attempt_id": "df911d68-7bd8-4477-95e7-ae70d95209ae", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005838000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.5936390429269522, "model_name": "gpt-5.6-luna", "output_tokens": 76, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:17.054582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005838000000000001, "created_at": "2026-09-17T03:24:17.054Z" }, { "sequence": 2929, "attempt_id": "4c0b40c6-8b85-4c29-889b-c293dc5bc4e5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00430325, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.031874821987003, "model_name": "muse-spark-1.3", "output_tokens": 274, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:17.221681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00430325, "created_at": "2026-09-17T03:24:17.221Z" }, { "sequence": 2930, "attempt_id": "6ff36dcd-0098-4bd8-b722-2d479712e9c8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00611, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 4.06434810499195, "model_name": "gemini-3.1-pro-preview", "output_tokens": 323, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:17.377163+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00611, "created_at": "2026-09-17T03:24:17.377Z" }, { "sequence": 2931, "attempt_id": "835806a0-562b-41c2-ba30-c40e03b11ec1", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.7476140210637823, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:17.529415+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:24:17.529Z" }, { "sequence": 2932, "attempt_id": "74b43cba-514f-40d9-8b61-09a8604abaa7", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.4206607500091195, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:18.637770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:24:18.637Z" }, { "sequence": 2933, "attempt_id": "e9a0260d-7ce9-4505-9a28-ebc99221aa8f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00204375, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.7940037329681218, "model_name": "gemini-3.8-flash", "output_tokens": 303, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:18.806151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00204375, "created_at": "2026-09-17T03:24:18.806Z" }, { "sequence": 2934, "attempt_id": "7dd68888-711d-487b-8f0d-0dc7baff059f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011058, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 7.557253677980043, "model_name": "gemini-3.1-pro-preview", "output_tokens": 729, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:19.048545+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011058, "created_at": "2026-09-17T03:24:19.048Z" }, { "sequence": 2935, "attempt_id": "47cec8ff-8d3f-4c05-be08-b3dd608b0ec6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0055825, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 7.070157867972739, "model_name": "muse-spark-1.3", "output_tokens": 565, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:19.537518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0055825, "created_at": "2026-09-17T03:24:19.537Z" }, { "sequence": 2936, "attempt_id": "16438409-0beb-4a77-aaac-297ff2361fb9", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004316, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.06521975598298, "model_name": "gemini-3.1-pro-preview", "output_tokens": 158, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:19.692762+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004316, "created_at": "2026-09-17T03:24:19.693Z" }, { "sequence": 2937, "attempt_id": "db6d9123-4891-4649-a884-8009cc87913b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.214155350928195, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:19.858514+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:24:19.860Z" }, { "sequence": 2938, "attempt_id": "d8c48656-ea59-4e93-8dc1-089f2ff09c25", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0778363180579618, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:20.012826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:24:20.013Z" }, { "sequence": 2939, "attempt_id": "3c3cca72-eba5-455c-9872-b9553664a0d5", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8395726840244606, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:20.698404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:24:20.698Z" }, { "sequence": 2940, "attempt_id": "ac4994bd-1842-4dd7-b81d-56d3ac04ae5b", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00192525, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.5246122379321605, "model_name": "gemini-3.8-flash", "output_tokens": 286, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:21.217709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00192525, "created_at": "2026-09-17T03:24:21.217Z" }, { "sequence": 2941, "attempt_id": "64661a59-b952-4435-8c92-231cbe553a5c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.5293723539216444, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:21.489153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:24:21.489Z" }, { "sequence": 2942, "attempt_id": "d9c0ae70-3e43-4066-912e-1fb06f95e2b0", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00415825, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.7730010619852692, "model_name": "muse-spark-1.2", "output_tokens": 234, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:21.938132+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00415825, "created_at": "2026-09-17T03:24:21.938Z" }, { "sequence": 2943, "attempt_id": "48d48d43-e6bc-4cff-a2df-fe9fafd1c5b5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1729047879343852, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:22.570344+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:24:22.570Z" }, { "sequence": 2944, "attempt_id": "838f7a7f-719a-4fa1-871c-6e492f58c9d3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.085568096023053, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:22.741942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:24:22.742Z" }, { "sequence": 2945, "attempt_id": "3066c993-d4f3-43ff-9ffb-8ec1c86f24df", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.8068501899251714, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:22.891554+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:24:22.891Z" }, { "sequence": 2946, "attempt_id": "b2664c7c-b112-43aa-887d-a5aacbf55be2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00468525, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.00944214197807, "model_name": "muse-spark-1.3", "output_tokens": 358, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:23.051006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00468525, "created_at": "2026-09-17T03:24:23.051Z" }, { "sequence": 2947, "attempt_id": "4ca56d86-9651-4941-b836-0ae0eece6c31", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00567, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.004715334973298, "model_name": "gemini-3.1-pro-preview", "output_tokens": 283, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:23.207299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00567, "created_at": "2026-09-17T03:24:23.207Z" }, { "sequence": 2948, "attempt_id": "b8610375-6e7b-43f2-8a49-293a09461129", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.268749544979073, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:23.372751+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:24:23.373Z" }, { "sequence": 2949, "attempt_id": "1fb41052-9b27-4690-b706-51fcbd7f6969", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 6.190238424111158, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:23.874700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:24:23.874Z" }, { "sequence": 2950, "attempt_id": "e91eeb0f-fc1d-44cf-82ce-7ba0d7842f2b", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.1030064440565184, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:24.154306+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:24:24.154Z" }, { "sequence": 2951, "attempt_id": "aa3dbb5c-fa4d-4935-9346-a1076c7acded", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00416675, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.32004954200238, "model_name": "muse-spark-1.2", "output_tokens": 236, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:24.693129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00416675, "created_at": "2026-09-17T03:24:24.693Z" }, { "sequence": 2952, "attempt_id": "0a3e8491-80d4-45f7-aa3b-c3b6cced6eb4", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00158775, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.8458366030827165, "model_name": "gemini-3.8-flash", "output_tokens": 196, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:24.892114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00158775, "created_at": "2026-09-17T03:24:24.892Z" }, { "sequence": 2953, "attempt_id": "6f4791b2-81a8-4a51-b541-b2e26db7bde8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1658582909731194, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:25.477164+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:24:25.477Z" }, { "sequence": 2954, "attempt_id": "bd1a46a5-bb4c-408f-9de3-af1db331185a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.9766826800769195, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:25.636128+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:24:25.636Z" }, { "sequence": 2955, "attempt_id": "89c6f825-bead-433c-a324-1c56be6d5ce3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00443875, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.3242592150345445, "model_name": "muse-spark-1.3", "output_tokens": 300, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:25.785733+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00443875, "created_at": "2026-09-17T03:24:25.785Z" }, { "sequence": 2956, "attempt_id": "630324c1-b2b9-4e2b-8123-ec4ec6f20078", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009491, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 8.433768459013663, "model_name": "muse-spark-1.2", "output_tokens": 1467, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:25.958319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009491, "created_at": "2026-09-17T03:24:25.958Z" }, { "sequence": 2957, "attempt_id": "2bd6ee29-6c9b-4900-8434-c3c875bff252", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.741872376995161, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:26.774313+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:24:26.774Z" }, { "sequence": 2958, "attempt_id": "ec50a3e1-db53-45c2-81ee-3975d0ad07dc", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003495, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0054808240383863, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:26.958853+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003495, "created_at": "2026-09-17T03:24:26.959Z" }, { "sequence": 2959, "attempt_id": "b02cc74e-d675-4a01-9304-0cc96db24845", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00128625, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.4081310339970514, "model_name": "gemini-3.8-flash", "output_tokens": 120, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:27.113952+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00128625, "created_at": "2026-09-17T03:24:27.114Z" }, { "sequence": 2960, "attempt_id": "79e8a72b-2315-472f-9ae5-25d98728bba6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.604983370983973, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:27.467188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:24:27.467Z" }, { "sequence": 2961, "attempt_id": "9bf9bc86-79fb-4f9b-8dca-9c032c6b2bff", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00429825, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.0557417949894443, "model_name": "muse-spark-1.2", "output_tokens": 274, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:28.169404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00429825, "created_at": "2026-09-17T03:24:28.169Z" }, { "sequence": 2962, "attempt_id": "d123c354-0d10-4f51-8ee7-145e663c6bac", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0439760079607368, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:28.330341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:24:28.331Z" }, { "sequence": 2963, "attempt_id": "91c2fa74-075b-4e4f-a863-8ec04bce8d5a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1950134029611945, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:28.492564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:24:28.492Z" }, { "sequence": 2964, "attempt_id": "1fd7da2f-ccde-4e56-ac1d-a8f4c0dc9e52", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006942, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.454103809082881, "model_name": "gemini-3.1-pro-preview", "output_tokens": 389, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:28.643874+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006942, "created_at": "2026-09-17T03:24:28.644Z" }, { "sequence": 2965, "attempt_id": "33689e9e-50c5-4746-add4-0ee9832fb9af", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7086975959828123, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:28.808088+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:24:28.808Z" }, { "sequence": 2966, "attempt_id": "62797e26-2bb4-446d-9108-a3702337f300", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9920257370686159, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:28.961369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:24:28.961Z" }, { "sequence": 2967, "attempt_id": "13b313f2-b5d5-4cbc-b538-21eb7875b9c4", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004396, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.8758477160008624, "model_name": "muse-spark-1.3", "output_tokens": 297, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:29.134923+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004396, "created_at": "2026-09-17T03:24:29.135Z" }, { "sequence": 2968, "attempt_id": "f7cca62a-e99f-4ce1-8a88-714bdcbcb8fe", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005158, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.238247242989019, "model_name": "gemini-3.1-pro-preview", "output_tokens": 244, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:29.294365+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005158, "created_at": "2026-09-17T03:24:29.294Z" }, { "sequence": 2969, "attempt_id": "f4549309-921a-400e-85a9-d75f9b686983", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7898848729673773, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:29.593350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:24:29.593Z" }, { "sequence": 2970, "attempt_id": "6bff5641-e152-4182-af46-20f0465e558f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00745525, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 12.652785988058895, "model_name": "muse-spark-1.3", "output_tokens": 988, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:29.874773+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00745525, "created_at": "2026-09-17T03:24:29.874Z" }, { "sequence": 2971, "attempt_id": "496d3151-e21c-4a36-83bd-fe66d88f1ffe", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01715525, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 17.428897729027085, "model_name": "muse-spark-1.2", "output_tokens": 3288, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:30.045129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01715525, "created_at": "2026-09-17T03:24:30.046Z" }, { "sequence": 2972, "attempt_id": "7766894a-0bb7-4bcd-8326-a178d89318bf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.9892844400601462, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:30.283949+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:24:30.284Z" }, { "sequence": 2973, "attempt_id": "4dc17f15-20e4-4fe5-9e38-9b488ac5981c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038315, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.1591214939253405, "model_name": "muse-spark-1.2", "output_tokens": 163, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:30.441813+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0038315, "created_at": "2026-09-17T03:24:30.442Z" }, { "sequence": 2974, "attempt_id": "42a96ac6-365e-404b-9c84-1e44bdec8692", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00138525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.6161276509519666, "model_name": "gemini-3.8-flash", "output_tokens": 146, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:30.595033+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00138525, "created_at": "2026-09-17T03:24:30.595Z" }, { "sequence": 2975, "attempt_id": "e6467c46-952b-4bf2-9b7c-80ddc420b669", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005034, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.115397223038599, "model_name": "gpt-5.6-terra", "output_tokens": 25, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:30.769521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005034, "created_at": "2026-09-17T03:24:30.769Z" }, { "sequence": 2976, "attempt_id": "6fbf103d-d812-4690-baa9-d4da1eb12d9c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.9233616959536448, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:30.921897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:24:30.922Z" }, { "sequence": 2977, "attempt_id": "f5af7be7-0bdd-451f-94fc-0558e15781dc", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 2.052714627003297, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:31.187951+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:24:31.188Z" }, { "sequence": 2978, "attempt_id": "070d7d8b-755c-4a4a-9ec8-1f7c0f0f34d9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.9958399450406432, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:31.437970+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:24:31.438Z" }, { "sequence": 2979, "attempt_id": "e3599079-50b0-49ad-a08e-ee1771e004b5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00477075, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.778705027070828, "model_name": "muse-spark-1.3", "output_tokens": 384, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:31.594696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00477075, "created_at": "2026-09-17T03:24:31.594Z" }, { "sequence": 2980, "attempt_id": "6d639378-909b-42f7-b079-2a2e1ee7a4c7", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003698, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.5228998919483274, "model_name": "gemini-3.1-pro-preview", "output_tokens": 122, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:31.853530+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003698, "created_at": "2026-09-17T03:24:31.853Z" }, { "sequence": 2981, "attempt_id": "099dceab-4067-4908-80f1-2c44f6b41fb4", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040445, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.276593278045766, "model_name": "muse-spark-1.2", "output_tokens": 214, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:32.046429+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0040445, "created_at": "2026-09-17T03:24:32.046Z" }, { "sequence": 2982, "attempt_id": "439c2b86-d9e3-4925-82e8-0c59fe7eaadd", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00124425, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.9255729339784011, "model_name": "gemini-3.8-flash", "output_tokens": 109, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:32.219877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00124425, "created_at": "2026-09-17T03:24:32.220Z" }, { "sequence": 2983, "attempt_id": "5bd4fdb8-a080-4068-b274-757d3998307a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1047187040094286, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:32.372372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:24:32.372Z" }, { "sequence": 2984, "attempt_id": "151913db-d94f-40f7-9bf6-f13687481a0f", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.6416736119426787, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:32.720520+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:24:32.720Z" }, { "sequence": 2985, "attempt_id": "294fbf44-513b-49d0-8702-6099413cd883", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00368, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 2.917451735935174, "model_name": "gemini-3.1-pro-preview", "output_tokens": 121, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:32.962897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00368, "created_at": "2026-09-17T03:24:32.963Z" }, { "sequence": 2986, "attempt_id": "02e71302-c34a-467c-9ead-fb7dfdcd472d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.9906035379972309, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:33.210778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:24:33.210Z" }, { "sequence": 2987, "attempt_id": "eb1aba02-7d19-409a-ac0c-542c3d7ff2b5", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001725, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.653746715048328, "model_name": "gemini-3.8-flash", "output_tokens": 218, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:33.668612+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001725, "created_at": "2026-09-17T03:24:33.668Z" }, { "sequence": 2988, "attempt_id": "b694d672-6d4c-48d4-97cc-a73c0d1e7bae", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.7905632550828159, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:34.670690+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:24:34.670Z" }, { "sequence": 2989, "attempt_id": "b16a5e84-710e-4c19-a654-4b9c96b316b2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005482, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.4402815649518743, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:34.833055+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005482, "created_at": "2026-09-17T03:24:34.834Z" }, { "sequence": 2990, "attempt_id": "8abc1c14-0e26-412d-8127-7edbb99f9cfe", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00465025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.514648661017418, "model_name": "muse-spark-1.2", "output_tokens": 328, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:35.048264+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00465025, "created_at": "2026-09-17T03:24:35.048Z" }, { "sequence": 2991, "attempt_id": "39eceec4-c888-4e2d-994f-f0b173399627", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00492, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 4.339462675037794, "model_name": "muse-spark-1.3", "output_tokens": 420, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:35.209797+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00492, "created_at": "2026-09-17T03:24:35.211Z" }, { "sequence": 2992, "attempt_id": "0684f076-2820-4539-bb01-b1758586612c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.158615182968788, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:35.373463+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:24:35.374Z" }, { "sequence": 2993, "attempt_id": "1121a8c1-e714-404e-8f30-817a98e5ca5e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005442, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 4.336499425000511, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:35.695764+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005442, "created_at": "2026-09-17T03:24:35.695Z" }, { "sequence": 2994, "attempt_id": "9fa5bd05-a1be-4430-bfd1-cde30b6d8e65", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005946, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 4.127287363982759, "model_name": "gpt-5.6-luna", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:35.871863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005946, "created_at": "2026-09-17T03:24:35.872Z" }, { "sequence": 2995, "attempt_id": "4f90146d-9b70-48a6-b875-db9f5ae9d4a9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.0586135110352188, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:36.268739+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:24:36.268Z" }, { "sequence": 2996, "attempt_id": "9c6b6d70-b4ea-4d6b-9b17-68fadff68c40", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00146625, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.448211501003243, "model_name": "gemini-3.8-flash", "output_tokens": 163, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:36.456164+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00146625, "created_at": "2026-09-17T03:24:36.456Z" }, { "sequence": 2997, "attempt_id": "bbca095d-7a99-4236-a939-06e6775b2eca", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005456, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.76258913311176, "model_name": "gemini-3.1-pro-preview", "output_tokens": 253, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:36.632674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005456, "created_at": "2026-09-17T03:24:36.632Z" }, { "sequence": 2998, "attempt_id": "11c374b1-e18d-43e1-9969-0b24e823dfe0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 3.589699932024814, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:37.420503+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:24:37.420Z" }, { "sequence": 2999, "attempt_id": "64b93566-680e-43eb-a88a-bd3f2912285a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 0.9534899670397863, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:37.589382+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004976000000000001, "created_at": "2026-09-17T03:24:37.590Z" }, { "sequence": 3000, "attempt_id": "75f161b9-2bae-46c7-923b-a28102a84029", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011256, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.5690638669766486, "model_name": "gpt-5.6-sol", "output_tokens": 82, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:38.604268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011256, "created_at": "2026-09-17T03:24:38.604Z" }, { "sequence": 3001, "attempt_id": "5ac9ad66-dfe6-4045-86ca-e3d60d7c8346", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.013740379945375, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:38.780159+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000374, "created_at": "2026-09-17T03:24:38.780Z" }, { "sequence": 3002, "attempt_id": "db030287-4ebe-4f80-8825-e2a6b7e3f58e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005996, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.404841198003851, "model_name": "gpt-5.6-terra", "output_tokens": 99, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:38.942806+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005996, "created_at": "2026-09-17T03:24:38.943Z" }, { "sequence": 3003, "attempt_id": "edf81fe0-b3c0-47fc-a1fe-94dad309d756", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00578675, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.696202198974788, "model_name": "muse-spark-1.2", "output_tokens": 616, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:39.233017+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00578675, "created_at": "2026-09-17T03:24:39.233Z" }, { "sequence": 3004, "attempt_id": "cf5fdb0d-1f05-4682-a95c-bdf659b663f7", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02669, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 3.5392844130983576, "model_name": "gpt-6-astra", "output_tokens": 53, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:39.411430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02669, "created_at": "2026-09-17T03:24:39.411Z" }, { "sequence": 3005, "attempt_id": "89f0302f-31f0-4617-b3ec-7eb06b205b2a", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004536, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.731966313091107, "model_name": "gemini-3.1-pro-preview", "output_tokens": 188, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:39.584733+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004536, "created_at": "2026-09-17T03:24:39.584Z" }, { "sequence": 3006, "attempt_id": "9743896c-0388-4ce1-866c-8c1f74799322", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1394957499578595, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:40.724548+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:24:40.724Z" }, { "sequence": 3007, "attempt_id": "4fe67eac-7977-4e28-a31a-47faf23173f9", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7646728709805757, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:40.890245+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:24:40.891Z" }, { "sequence": 3008, "attempt_id": "268d59cd-4d91-4375-a783-9543aebddc5a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1135877210181206, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:41.046284+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:24:41.046Z" }, { "sequence": 3009, "attempt_id": "3b836275-42a0-47c5-9b34-18fd827134d6", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00463125, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 3.8937248210422695, "model_name": "gemini-3.8-flash", "output_tokens": 1004, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:41.483450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00463125, "created_at": "2026-09-17T03:24:41.483Z" }, { "sequence": 3010, "attempt_id": "92719f0d-6f7b-4dd8-a96e-6b3cebe74615", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007247, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 9.19980038003996, "model_name": "muse-spark-1.3", "output_tokens": 939, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:41.728151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007247, "created_at": "2026-09-17T03:24:41.728Z" }, { "sequence": 3011, "attempt_id": "a0c49bbc-7b6e-4da3-8a68-4f83ede9711f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.3734541659941897, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:41.890702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:24:41.892Z" }, { "sequence": 3012, "attempt_id": "1308da16-04fb-42b5-8532-be84178f6108", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7556012640707195, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:42.057623+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:24:42.059Z" }, { "sequence": 3013, "attempt_id": "4e028b33-4de6-4fed-85a0-2e933fc2f516", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00121425, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.4702460159314796, "model_name": "gemini-3.8-flash", "output_tokens": 97, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:42.690164+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00121425, "created_at": "2026-09-17T03:24:42.690Z" }, { "sequence": 3014, "attempt_id": "cba5f2d7-fcc7-499d-a7e6-2d246f584e48", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003348, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.514783123973757, "model_name": "gemini-3.1-pro-preview", "output_tokens": 90, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:43.405397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003348, "created_at": "2026-09-17T03:24:43.405Z" }, { "sequence": 3015, "attempt_id": "7ec13e07-cbbd-43c9-bd59-51ce78100b08", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00393775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.9811276539694518, "model_name": "muse-spark-1.3", "output_tokens": 183, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:43.628134+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00393775, "created_at": "2026-09-17T03:24:43.628Z" }, { "sequence": 3016, "attempt_id": "9e3a0ea6-7128-4282-a533-3f65e9aa5bc2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00467725, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.2796558931004256, "model_name": "muse-spark-1.2", "output_tokens": 357, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:44.170698+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00467725, "created_at": "2026-09-17T03:24:44.170Z" }, { "sequence": 3017, "attempt_id": "4844d824-973e-421a-9008-a6d79cd8c28f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005616, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.5025471670087427, "model_name": "gpt-5.6-terra", "output_tokens": 57, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:44.362377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005616, "created_at": "2026-09-17T03:24:44.362Z" }, { "sequence": 3018, "attempt_id": "4c99da58-76ae-4b24-9bb9-564ce9a42e46", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.7039277120493352, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:44.761865+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:24:44.762Z" }, { "sequence": 3019, "attempt_id": "4b88e0e7-7890-4b36-8947-c09ba949a98b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011364, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.535183506901376, "model_name": "gpt-5.6-sol", "output_tokens": 75, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:44.925515+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011364, "created_at": "2026-09-17T03:24:44.925Z" }, { "sequence": 3020, "attempt_id": "f0c7cad8-7d0d-407b-bf49-c8a9cdec09cb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0057145, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 9.578903277986683, "model_name": "muse-spark-1.3", "output_tokens": 599, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:45.101068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0057145, "created_at": "2026-09-17T03:24:45.101Z" }, { "sequence": 3021, "attempt_id": "7a07926c-47e1-429b-b09f-6b1a77bf02f6", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8845391189679503, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:45.409003+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003789, "created_at": "2026-09-17T03:24:45.409Z" }, { "sequence": 3022, "attempt_id": "d347e237-2232-4d2f-adb5-db3f53234967", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000594, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.2220533039653674, "model_name": "gpt-5.6-luna", "output_tokens": 84, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:45.789567+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000594, "created_at": "2026-09-17T03:24:45.789Z" }, { "sequence": 3023, "attempt_id": "71f2262b-c1c9-43e5-ba20-4b5a70bfdac9", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": true, "cost_usd": 0.072178, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 9.201464507961646, "model_name": "gemini-3.1-pro-preview", "output_tokens": 893, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:45.997289+00:00", "request_attempts": 2, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 1, "valid": true, "within_bands": null }, "cost_usd": 0.072178, "created_at": "2026-09-17T03:24:45.997Z" }, { "sequence": 3024, "attempt_id": "e01f8c02-631e-4496-a60b-b5bde04bc382", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.7235110480105504, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:46.983274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:24:46.983Z" }, { "sequence": 3025, "attempt_id": "d90704e8-2718-4c9a-ace8-ec3616a656d9", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0020535, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.693159295944497, "model_name": "gemini-3.8-flash", "output_tokens": 305, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:47.149115+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0020535, "created_at": "2026-09-17T03:24:47.149Z" }, { "sequence": 3026, "attempt_id": "aa620e6d-3ca2-4bd9-8a31-d8079fbf67ce", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4192331930389628, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:47.307999+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:24:47.308Z" }, { "sequence": 3027, "attempt_id": "932caa97-50de-46b0-9f0a-5920660620b0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005376, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4385217500384897, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:47.482988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005376, "created_at": "2026-09-17T03:24:47.483Z" }, { "sequence": 3028, "attempt_id": "15b5ad17-3ef5-4883-b8d8-00dda9bc878c", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-002", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.8124513559741899, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:48.295794+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003764, "created_at": "2026-09-17T03:24:48.296Z" }, { "sequence": 3029, "attempt_id": "38cd76ca-5ac6-48ad-8a3c-467cb336b148", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.242056049988605, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:48.459040+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:24:48.459Z" }, { "sequence": 3030, "attempt_id": "4a8177e7-ebbd-45dc-9bb7-5b8b930cf3fd", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008594, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.614269832964055, "model_name": "gemini-3.1-pro-preview", "output_tokens": 514, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:50.417526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008594, "created_at": "2026-09-17T03:24:50.417Z" }, { "sequence": 3031, "attempt_id": "3a23150a-1971-411a-8309-678b53cb4a6e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00779775, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 6.222538274945691, "model_name": "muse-spark-1.2", "output_tokens": 1068, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:51.323947+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00779775, "created_at": "2026-09-17T03:24:51.324Z" }, { "sequence": 3032, "attempt_id": "b2c56e7e-0d5d-415e-be5b-0b1a891bc05c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005794, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.205903830006719, "model_name": "gemini-3.1-pro-preview", "output_tokens": 289, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:51.503539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005794, "created_at": "2026-09-17T03:24:51.505Z" }, { "sequence": 3033, "attempt_id": "8e706fd5-b6e1-42b3-8c02-5d2614b847d3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012782, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 12.715429529896937, "model_name": "muse-spark-1.2", "output_tokens": 2259, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:51.667341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012782, "created_at": "2026-09-17T03:24:51.667Z" }, { "sequence": 3034, "attempt_id": "e3ed5d2e-777c-47c9-a786-a0430341590c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 3.003540235920809, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:51.823319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:24:51.823Z" }, { "sequence": 3035, "attempt_id": "b50b6967-7cd5-48c1-8e66-e4b6b4ae37cd", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049635, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.348170951008797, "model_name": "gemini-3.8-flash", "output_tokens": 1091, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:51.989169+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049635, "created_at": "2026-09-17T03:24:51.990Z" }, { "sequence": 3036, "attempt_id": "3817f86b-7ac8-4ca5-83ab-d18d3fe3c71f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 0.9333124560071155, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:52.601001+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004944, "created_at": "2026-09-17T03:24:52.601Z" }, { "sequence": 3037, "attempt_id": "023cadcc-52cb-4700-9590-0df3f3950414", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010892, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.050392927019857, "model_name": "gpt-5.6-sol", "output_tokens": 67, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:52.766973+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010892, "created_at": "2026-09-17T03:24:52.767Z" }, { "sequence": 3038, "attempt_id": "6dc87379-d694-4bb7-af26-170ab34a7286", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.3362305740593001, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:52.930494+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005436, "created_at": "2026-09-17T03:24:52.930Z" }, { "sequence": 3039, "attempt_id": "80c0c521-d8c0-4160-be81-5fbbb875ba58", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-002", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.1911816929932684, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:53.353804+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000368, "created_at": "2026-09-17T03:24:53.353Z" }, { "sequence": 3040, "attempt_id": "f832c6c7-ad2d-4535-878f-35ed8623a3f0", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00241125, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 2.3341858809581026, "model_name": "gemini-3.8-flash", "output_tokens": 416, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:54.323709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00241125, "created_at": "2026-09-17T03:24:54.323Z" }, { "sequence": 3041, "attempt_id": "9c2248e8-b48f-4591-9686-0903fcffdce1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011424, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.1914610990788788, "model_name": "gpt-5.6-sol", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:55.721700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011424, "created_at": "2026-09-17T03:24:55.721Z" }, { "sequence": 3042, "attempt_id": "f7ac12b8-885b-44a2-b3b4-dd5be45a7e1b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0073515, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 11.000559662934393, "model_name": "muse-spark-1.3", "output_tokens": 963, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:55.926368+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0073515, "created_at": "2026-09-17T03:24:55.926Z" }, { "sequence": 3043, "attempt_id": "7d9362a5-234a-4f45-9f10-4cf020907d5b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00529475, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 3.0216475769411772, "model_name": "muse-spark-1.2", "output_tokens": 502, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:56.088074+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00529475, "created_at": "2026-09-17T03:24:56.088Z" }, { "sequence": 3044, "attempt_id": "326b8d32-d442-4e67-b1d1-50ad6d4ec883", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02716, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.953108998015523, "model_name": "gpt-6-astra", "output_tokens": 50, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:56.254792+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02716, "created_at": "2026-09-17T03:24:56.254Z" }, { "sequence": 3045, "attempt_id": "cf09cba3-09f6-4b0f-8d05-0a9c0b8cb6cf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005868, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.141844204044901, "model_name": "gpt-5.6-terra", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:56.629034+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005868, "created_at": "2026-09-17T03:24:56.629Z" }, { "sequence": 3046, "attempt_id": "e9bb398f-0f61-48e5-9ef0-042e852c6813", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008234, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.230458915000781, "model_name": "gemini-3.1-pro-preview", "output_tokens": 497, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:57.214788+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008234, "created_at": "2026-09-17T03:24:57.214Z" }, { "sequence": 3047, "attempt_id": "a91c88d7-99e4-41c6-9278-d76c3827103d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000618, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7791024670004845, "model_name": "gpt-5.6-luna", "output_tokens": 104, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:57.664333+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000618, "created_at": "2026-09-17T03:24:57.664Z" }, { "sequence": 3048, "attempt_id": "7d529c07-8030-4f5b-a071-ccb2be77c5ff", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.2986678979359567, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:57.844397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:24:57.844Z" }, { "sequence": 3049, "attempt_id": "e7790fa6-d019-4e50-8d62-5a686cbe29d3", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002706, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.6345270660240203, "model_name": "gemini-3.8-flash", "output_tokens": 479, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:59.046509+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002706, "created_at": "2026-09-17T03:24:59.046Z" }, { "sequence": 3050, "attempt_id": "40c6bf23-0254-4a0a-a31e-62b36315a97b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.566450126003474, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:59.411226+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:24:59.411Z" }, { "sequence": 3051, "attempt_id": "387feca2-a0bd-411c-9319-b6cd3e79c097", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7061888530151919, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:59.716113+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:24:59.716Z" }, { "sequence": 3052, "attempt_id": "ca2937fe-dabf-41d6-9ff5-3960fcf83cd9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00604275, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 7.286079034907743, "model_name": "muse-spark-1.3", "output_tokens": 678, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:00.053420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00604275, "created_at": "2026-09-17T03:25:00.053Z" }, { "sequence": 3053, "attempt_id": "2bf6b5ea-1161-4626-a7ac-7aa2c9fc5b65", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0440604130271822, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:00.256598+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:25:00.256Z" }, { "sequence": 3054, "attempt_id": "b13d2e58-aa67-45c2-81de-393657114be9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8400070160860196, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:01.256136+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:25:01.256Z" }, { "sequence": 3055, "attempt_id": "f8a92857-83ac-47bd-b99b-ca8d8827d0c5", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012465, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.2541671840008348, "model_name": "gemini-3.8-flash", "output_tokens": 105, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:01.469591+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0012465, "created_at": "2026-09-17T03:25:01.469Z" }, { "sequence": 3056, "attempt_id": "04a06a29-bbd1-4e8e-a456-f57dec8a0a58", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01281175, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 22.86072623205837, "model_name": "muse-spark-1.3", "output_tokens": 2266, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:01.646420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01281175, "created_at": "2026-09-17T03:25:01.646Z" }, { "sequence": 3057, "attempt_id": "f02484b1-8cb5-40fc-a491-3f2ba9560940", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008438, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.96843675407581, "model_name": "gemini-3.1-pro-preview", "output_tokens": 501, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:02.056867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008438, "created_at": "2026-09-17T03:25:02.057Z" }, { "sequence": 3058, "attempt_id": "4b3bc344-58a5-4392-9831-e9d3aa9b2173", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.7079283150378615, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:02.283256+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:25:02.283Z" }, { "sequence": 3059, "attempt_id": "8d0ab118-6012-42e1-8b28-1f46a5d9957b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004171, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.3058922209311277, "model_name": "muse-spark-1.2", "output_tokens": 237, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:02.952672+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004171, "created_at": "2026-09-17T03:25:02.952Z" }, { "sequence": 3060, "attempt_id": "fdf758ea-c9b3-4903-81bd-f2cdcf070f4d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0736963750096038, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:03.294151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:25:03.294Z" }, { "sequence": 3061, "attempt_id": "c14210c7-bbbe-4781-b3a1-6195ecf4c8e8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003714, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.927906552911736, "model_name": "gemini-3.1-pro-preview", "output_tokens": 120, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:03.818863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003714, "created_at": "2026-09-17T03:25:03.819Z" }, { "sequence": 3062, "attempt_id": "69c5400a-cdbc-4a96-8a70-a3fb8f00bf1c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00414975, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.4605207630665973, "model_name": "muse-spark-1.3", "output_tokens": 232, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:03.982897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00414975, "created_at": "2026-09-17T03:25:03.983Z" }, { "sequence": 3063, "attempt_id": "0d716ed9-5444-40fe-be66-88bf30ef1c90", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.6086741089820862, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:04.143609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:25:04.143Z" }, { "sequence": 3064, "attempt_id": "be956b66-4ba1-42ad-960f-7629beaba6fc", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012105, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.1641646820353344, "model_name": "gemini-3.8-flash", "output_tokens": 96, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:05.147434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0012105, "created_at": "2026-09-17T03:25:05.147Z" }, { "sequence": 3065, "attempt_id": "a622c89f-5d9d-481c-9a63-18000a9c47f2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.302706054993905, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:05.430360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:25:05.430Z" }, { "sequence": 3066, "attempt_id": "aed8e89a-3a13-4783-8e86-f6676f671f06", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.4630061640636995, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:05.606891+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:25:05.607Z" }, { "sequence": 3067, "attempt_id": "52a1543e-3136-4c66-81c9-153b6a1fd608", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 4.956211532000452, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:06.764863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:25:06.765Z" }, { "sequence": 3068, "attempt_id": "f88bed61-8d87-40f4-8f73-5f70ce655152", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004027, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6848621719982475, "model_name": "muse-spark-1.2", "output_tokens": 204, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:06.997031+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004027, "created_at": "2026-09-17T03:25:06.997Z" }, { "sequence": 3069, "attempt_id": "30fd60bd-20f2-4f0a-86a3-0400da23a222", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.6962999389506876, "model_name": "gemini-3.1-pro-preview", "output_tokens": 236, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:07.160553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0051, "created_at": "2026-09-17T03:25:07.160Z" }, { "sequence": 3070, "attempt_id": "2ad17751-5929-45ac-94df-d71446e21dce", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.6074632800882682, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:07.375743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:25:07.375Z" }, { "sequence": 3071, "attempt_id": "7fae146a-39d9-4106-9e16-eb93a6ead748", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.846642707940191, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:07.537783+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:25:07.538Z" }, { "sequence": 3072, "attempt_id": "646c5bc5-df7d-42ab-96ef-fcb0809bf374", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01141875, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.537435580044985, "model_name": "muse-spark-1.2", "output_tokens": 1920, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:07.916065+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01141875, "created_at": "2026-09-17T03:25:07.916Z" }, { "sequence": 3073, "attempt_id": "5853baea-cfde-43ac-a41e-55933fcf8924", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.755953288055025, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:08.081245+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:25:08.082Z" }, { "sequence": 3074, "attempt_id": "10fe8efd-20fa-4376-8aca-521ba04a0ac5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00435425, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.771300456020981, "model_name": "muse-spark-1.3", "output_tokens": 281, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:08.242913+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00435425, "created_at": "2026-09-17T03:25:08.243Z" }, { "sequence": 3075, "attempt_id": "629f21c2-025f-4206-8f1b-d97e97d7a1a3", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.7094053089385852, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:08.422924+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003683, "created_at": "2026-09-17T03:25:08.423Z" }, { "sequence": 3076, "attempt_id": "8efd3a76-ec9f-4d91-9606-830c87800fc1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01872825, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 20.348303647944704, "model_name": "muse-spark-1.2", "output_tokens": 3654, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:25:08.807671+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01872825, "created_at": "2026-09-17T03:25:08.807Z" }, { "sequence": 3077, "attempt_id": "61d60605-fdac-4805-b29d-bbf20be1a2de", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0014595, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.5725890630856156, "model_name": "gemini-3.8-flash", "output_tokens": 162, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:09.110684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0014595, "created_at": "2026-09-17T03:25:09.110Z" }, { "sequence": 3078, "attempt_id": "e9868aa1-5121-45ac-8a71-7661dd1f940c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00533, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.9649396570166573, "model_name": "gpt-5.6-terra", "output_tokens": 46, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:09.906762+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00533, "created_at": "2026-09-17T03:25:09.906Z" }, { "sequence": 3079, "attempt_id": "da206118-8a75-413a-a215-c239797d773f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.6931242380524054, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:10.277347+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:25:10.277Z" }, { "sequence": 3080, "attempt_id": "f220cfa9-48f2-4715-963f-081b7a9fa696", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.7831757320091128, "model_name": "gpt-5.6-terra", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:10.754619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T03:25:10.754Z" }, { "sequence": 3081, "attempt_id": "9be819fd-4e80-4e1d-9e51-5ec87a998391", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006018000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.6884302790276706, "model_name": "gpt-5.6-luna", "output_tokens": 91, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:10.962864+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006018000000000001, "created_at": "2026-09-17T03:25:10.963Z" }, { "sequence": 3082, "attempt_id": "f2896d65-14f1-424c-8b8f-3e8c7425e62a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.9979225450661033, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:11.416933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:25:11.417Z" }, { "sequence": 3083, "attempt_id": "efbd43c5-c4a8-4b17-b771-7c1b801559ee", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.873080765013583, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:11.790264+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:25:11.790Z" }, { "sequence": 3084, "attempt_id": "70b5372a-7ef5-4ed8-879f-779bf74be0bb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006163, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.939325519022532, "model_name": "muse-spark-1.2", "output_tokens": 706, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:12.182556+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006163, "created_at": "2026-09-17T03:25:12.182Z" }, { "sequence": 3085, "attempt_id": "bd93197c-ae43-4990-850c-bd862b518320", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0024075, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.7858624089276418, "model_name": "gemini-3.8-flash", "output_tokens": 400, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:12.346822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0024075, "created_at": "2026-09-17T03:25:12.347Z" }, { "sequence": 3086, "attempt_id": "858bb771-0367-4af2-ad66-d468a2bf63c8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006616, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 5.460470407037064, "model_name": "gemini-3.1-pro-preview", "output_tokens": 362, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:12.782892+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006616, "created_at": "2026-09-17T03:25:12.783Z" }, { "sequence": 3087, "attempt_id": "1aa72f4b-9448-4af3-a764-305a08f0c93e", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1210955979768187, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:13.468253+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:25:13.468Z" }, { "sequence": 3088, "attempt_id": "5fc66edf-8ead-4975-9ae6-6af7d90c596d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.281080739106983, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:14.230091+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:25:14.230Z" }, { "sequence": 3089, "attempt_id": "40f61743-e97a-451d-9231-ef1d8ab17ad7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9666324650170282, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:14.487658+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:25:14.487Z" }, { "sequence": 3090, "attempt_id": "540ed51d-7ae7-4e60-8d98-325b3c64f56b", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006932, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 5.061335948063061, "model_name": "gemini-3.1-pro-preview", "output_tokens": 376, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:15.145134+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006932, "created_at": "2026-09-17T03:25:15.145Z" }, { "sequence": 3091, "attempt_id": "040e3398-bc50-42ac-9888-92cc1e33d2a6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 3.19808132504113, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:15.312383+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:25:15.312Z" }, { "sequence": 3092, "attempt_id": "ecd88d9f-52ab-48c3-98a8-305b6105c486", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006044, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 7.618747737957165, "model_name": "muse-spark-1.3", "output_tokens": 678, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:15.700331+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006044, "created_at": "2026-09-17T03:25:15.700Z" }, { "sequence": 3093, "attempt_id": "6375fcbf-0356-46d8-bfef-2ad51bba93b4", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003552, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.2309671400580555, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:15.897506+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003552, "created_at": "2026-09-17T03:25:15.897Z" }, { "sequence": 3094, "attempt_id": "c8bfd032-c991-4658-b8e0-0dc1bccb6b21", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00115425, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.457596594002098, "model_name": "gemini-3.8-flash", "output_tokens": 81, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:16.853832+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00115425, "created_at": "2026-09-17T03:25:16.854Z" }, { "sequence": 3095, "attempt_id": "f7625781-c2d2-46c7-8284-6cbe714d1283", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003168, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.292831468046643, "model_name": "gemini-3.1-pro-preview", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:17.021796+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003168, "created_at": "2026-09-17T03:25:17.022Z" }, { "sequence": 3096, "attempt_id": "e003565f-8c34-4e9d-ab7c-39d91095382c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.1351974779972807, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:17.205965+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:25:17.206Z" }, { "sequence": 3097, "attempt_id": "e19c9eec-9fb1-4de5-9ef9-981f49c82092", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6773004749557003, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:17.540895+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:25:17.541Z" }, { "sequence": 3098, "attempt_id": "21b349da-80e6-487e-96f4-b58b16589f7b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004197, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.3229895969852805, "model_name": "muse-spark-1.3", "output_tokens": 244, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:17.717115+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004197, "created_at": "2026-09-17T03:25:17.718Z" }, { "sequence": 3099, "attempt_id": "ac59c937-8cb7-444e-bde4-88f320fe52cf", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00427775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.3678953329799697, "model_name": "muse-spark-1.2", "output_tokens": 263, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:17.881628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00427775, "created_at": "2026-09-17T03:25:17.881Z" }, { "sequence": 3100, "attempt_id": "6aa5897d-a591-4065-8fee-4fdafffb8107", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00754025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 6.342727012000978, "model_name": "muse-spark-1.2", "output_tokens": 1008, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:18.047119+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00754025, "created_at": "2026-09-17T03:25:18.048Z" }, { "sequence": 3101, "attempt_id": "6445c783-fe2f-4bed-9cf4-6eb7d086f233", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.6710527880350128, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:18.553001+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:25:18.553Z" }, { "sequence": 3102, "attempt_id": "61e9a952-66ef-4960-84d6-9ed6a162a2d8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005664, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.5608075649943203, "model_name": "gpt-5.6-luna", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:18.745467+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005664, "created_at": "2026-09-17T03:25:18.745Z" }, { "sequence": 3103, "attempt_id": "315116b1-0920-437a-84d6-e40f33d23118", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.0356965459650382, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:19.057815+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T03:25:19.057Z" }, { "sequence": 3104, "attempt_id": "ab89d42a-91d7-41cb-943c-1c8ca36d6928", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0020925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.855817079078406, "model_name": "gemini-3.8-flash", "output_tokens": 328, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:19.573319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0020925, "created_at": "2026-09-17T03:25:19.573Z" }, { "sequence": 3105, "attempt_id": "51dc1cd1-9e73-4548-bdb0-b6afd078bb1d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0402068730909377, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:19.948682+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:25:19.948Z" }, { "sequence": 3106, "attempt_id": "ffb27878-0092-429c-8e96-bbad31543630", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.7836950049968436, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:20.535045+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:25:20.535Z" }, { "sequence": 3107, "attempt_id": "8d8efc98-abe2-4914-8bf7-6c465cffbcaf", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.8263020509621128, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:20.702188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:25:20.702Z" }, { "sequence": 3108, "attempt_id": "e10357dd-6f64-4d5e-b0b1-632381af9489", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7675179750658572, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:21.633739+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:25:21.633Z" }, { "sequence": 3109, "attempt_id": "aa45d881-5cdc-4574-82e1-cf257e2e39f1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00830525, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 10.706701286020689, "model_name": "muse-spark-1.3", "output_tokens": 1188, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:21.829970+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00830525, "created_at": "2026-09-17T03:25:21.830Z" }, { "sequence": 3110, "attempt_id": "a5f989d3-4fb0-45db-b726-4ce5bb533216", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001389, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.2497513659764081, "model_name": "gemini-3.8-flash", "output_tokens": 147, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:21.994177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001389, "created_at": "2026-09-17T03:25:21.994Z" }, { "sequence": 3111, "attempt_id": "d9ef7ba8-a2f8-4772-b106-86a7780b6de5", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006104, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.660939791006967, "model_name": "gemini-3.1-pro-preview", "output_tokens": 317, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:22.159158+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006104, "created_at": "2026-09-17T03:25:22.159Z" }, { "sequence": 3112, "attempt_id": "bd0a61d0-b79d-4255-b9d5-986a544582c4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.000796628999524, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:22.322928+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:25:22.323Z" }, { "sequence": 3113, "attempt_id": "3f5cdf43-24e5-4a9b-905d-5b66ed7113ee", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003482, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.0122045759344473, "model_name": "gemini-3.1-pro-preview", "output_tokens": 104, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:23.125696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003482, "created_at": "2026-09-17T03:25:23.125Z" }, { "sequence": 3114, "attempt_id": "c1de09ae-ea3f-4323-b787-6cb43f743a85", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003993, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.6066754390485585, "model_name": "muse-spark-1.2", "output_tokens": 201, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:23.601191+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003993, "created_at": "2026-09-17T03:25:23.601Z" }, { "sequence": 3115, "attempt_id": "38cb1ecf-8a5b-4f5b-98b9-45f203a313f5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01288075, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 26.9692703000037, "model_name": "muse-spark-1.3", "output_tokens": 2264, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:23.768918+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01288075, "created_at": "2026-09-17T03:25:23.769Z" }, { "sequence": 3116, "attempt_id": "3574d114-4d25-4641-bdc2-dd325449ad1d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.5060487389564514, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:23.947308+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:25:23.947Z" }, { "sequence": 3117, "attempt_id": "7a55f2a8-a6e4-4ed8-8ffc-b60132463527", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.65635155106429, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:24.105254+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:25:24.105Z" }, { "sequence": 3118, "attempt_id": "f42e1310-33eb-436e-aea7-7bb27e8dfb7d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00845475, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.04805161699187, "model_name": "muse-spark-1.2", "output_tokens": 1242, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:24.273224+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00845475, "created_at": "2026-09-17T03:25:24.273Z" }, { "sequence": 3119, "attempt_id": "dee08b51-5522-4ce3-989b-ba3b4ad06ad4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00546, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6617704869713634, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:24.431378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00546, "created_at": "2026-09-17T03:25:24.431Z" }, { "sequence": 3120, "attempt_id": "95d7fe63-275a-4d02-b20f-13e6a99db9e7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00579, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.410999399027787, "model_name": "muse-spark-1.3", "output_tokens": 615, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:24.614090+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00579, "created_at": "2026-09-17T03:25:24.615Z" }, { "sequence": 3121, "attempt_id": "c87e9790-3a25-454b-86ad-a84a57215480", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.8059873620513827, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:25.074000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036, "created_at": "2026-09-17T03:25:25.074Z" }, { "sequence": 3122, "attempt_id": "f2a4dbab-4df3-4128-93ba-485286ca00a6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044435, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.7087591529125348, "model_name": "muse-spark-1.3", "output_tokens": 307, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:25.521689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0044435, "created_at": "2026-09-17T03:25:25.521Z" }, { "sequence": 3123, "attempt_id": "79bc8bb1-1ec4-4e4e-bed3-16c1cac200ba", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001995, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.8888942629564553, "model_name": "gemini-3.8-flash", "output_tokens": 302, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:25.836508+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001995, "created_at": "2026-09-17T03:25:25.836Z" }, { "sequence": 3124, "attempt_id": "2d7a78ce-1dde-4d20-8f5d-6921ace1bfe4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.6872070520184934, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:26.301617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:25:26.301Z" }, { "sequence": 3125, "attempt_id": "f22b5dfb-a615-4eed-8914-b29f470b21c3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.0804049379657954, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:26.768752+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004944, "created_at": "2026-09-17T03:25:26.768Z" }, { "sequence": 3126, "attempt_id": "48a9f2d3-f5bf-45dd-ba52-a46ea52d1906", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.5382910799235106, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:26.935242+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005436, "created_at": "2026-09-17T03:25:26.935Z" }, { "sequence": 3127, "attempt_id": "497fb8ee-b4bf-45ab-ad92-bc192d8a09ff", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010712, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.210258510080166, "model_name": "gpt-5.6-sol", "output_tokens": 58, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:27.113589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010712, "created_at": "2026-09-17T03:25:27.113Z" }, { "sequence": 3128, "attempt_id": "942f7ab7-9331-43a7-a289-697ed1109af7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005616, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 4.105023059993982, "model_name": "gpt-5.6-luna", "output_tokens": 68, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:27.397321+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005616, "created_at": "2026-09-17T03:25:27.397Z" }, { "sequence": 3129, "attempt_id": "176aff27-ab35-4d8a-af02-d15dea53a9a1", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00386, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.9940413139993325, "model_name": "gemini-3.1-pro-preview", "output_tokens": 130, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:27.763289+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00386, "created_at": "2026-09-17T03:25:27.763Z" }, { "sequence": 3130, "attempt_id": "c34b21af-ee8e-4903-9c87-69e23369605f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-003", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.9511572680203244, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:27.932516+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003555, "created_at": "2026-09-17T03:25:27.932Z" }, { "sequence": 3131, "attempt_id": "9beb2353-992e-4c86-a812-0ca44e860b01", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0060025, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.058040195028298, "model_name": "muse-spark-1.2", "output_tokens": 665, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:28.666934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0060025, "created_at": "2026-09-17T03:25:28.667Z" }, { "sequence": 3132, "attempt_id": "c78b12e9-2eaa-49b7-8e42-970f2e0eee3f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00438525, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 1.8540575100341812, "model_name": "muse-spark-1.2", "output_tokens": 288, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:29.130556+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00438525, "created_at": "2026-09-17T03:25:29.130Z" }, { "sequence": 3133, "attempt_id": "7c919ae7-b6c4-4878-bad3-cc0593b62b7a", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7147251169662923, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:29.298405+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02454, "created_at": "2026-09-17T03:25:29.298Z" }, { "sequence": 3134, "attempt_id": "2dbf3fbe-0965-425f-8382-e1116e949777", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00231375, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 3.2448490489041433, "model_name": "gemini-3.8-flash", "output_tokens": 390, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:29.711903+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00231375, "created_at": "2026-09-17T03:25:29.712Z" }, { "sequence": 3135, "attempt_id": "0bbf39f9-8071-4a54-9fac-16453c5a5b4f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005536, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7037369899917394, "model_name": "gpt-5.6-terra", "output_tokens": 64, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:29.893107+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005536, "created_at": "2026-09-17T03:25:29.894Z" }, { "sequence": 3136, "attempt_id": "bea9dd83-cfe4-4d65-bc78-21c94f8704fb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.0524746279697865, "model_name": "gpt-5.6-sol", "output_tokens": 66, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:30.060336+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010856, "created_at": "2026-09-17T03:25:30.060Z" }, { "sequence": 3137, "attempt_id": "2caa2814-d4b4-46bc-a1e0-94bc8064bae2", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 1.003940571914427, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:30.897442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003668, "created_at": "2026-09-17T03:25:30.897Z" }, { "sequence": 3138, "attempt_id": "888aeb64-f931-4227-a025-4459b46e2cb8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008942, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 6.214299532002769, "model_name": "gemini-3.1-pro-preview", "output_tokens": 556, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:32.227676+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008942, "created_at": "2026-09-17T03:25:32.227Z" }, { "sequence": 3139, "attempt_id": "09b97496-cb96-4c92-98fb-edf3d77aae8e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003792, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 3.3399625789606944, "model_name": "gemini-3.8-flash", "output_tokens": 785, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:32.802017+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003792, "created_at": "2026-09-17T03:25:32.802Z" }, { "sequence": 3140, "attempt_id": "df4d32ee-534d-497a-b104-c2e9707c4208", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.7649094460066408, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:32.971030+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:25:32.971Z" }, { "sequence": 3141, "attempt_id": "ef6ca71c-a6db-441f-bb5a-c64a02fa43a5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005824, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 5.006178608047776, "model_name": "gpt-5.6-luna", "output_tokens": 88, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:33.841657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005824, "created_at": "2026-09-17T03:25:33.841Z" }, { "sequence": 3142, "attempt_id": "1b1654b5-c566-4244-b65c-a2f7ec25acfd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005264, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.2805293669225648, "model_name": "gpt-5.6-terra", "output_tokens": 42, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:34.251884+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005264, "created_at": "2026-09-17T03:25:34.252Z" }, { "sequence": 3143, "attempt_id": "11dc0232-85e5-4d47-b9bf-fe4cb5af9ca1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.8537091000471264, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:34.418570+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:25:34.418Z" }, { "sequence": 3144, "attempt_id": "08100313-d529-4b6d-a7f7-46103abe32ce", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.6037661120062694, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:34.742569+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:25:34.742Z" }, { "sequence": 3145, "attempt_id": "19b98fbb-3421-46b8-b218-67e4bb25c211", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8582249619066715, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:35.441732+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:25:35.441Z" }, { "sequence": 3146, "attempt_id": "f5f0e5df-502c-4a80-bc03-cfa1061e3280", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0060555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 9.509411437087692, "model_name": "muse-spark-1.3", "output_tokens": 681, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:36.623360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0060555, "created_at": "2026-09-17T03:25:36.623Z" }, { "sequence": 3147, "attempt_id": "4d071f50-7b0e-4c21-b572-2e61781345bd", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01227, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 8.085181782022119, "model_name": "gemini-3.1-pro-preview", "output_tokens": 834, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:37.383928+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01227, "created_at": "2026-09-17T03:25:37.384Z" }, { "sequence": 3148, "attempt_id": "49b19a50-a236-476f-bb53-6a156656c217", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0027615, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 2.994213673984632, "model_name": "gemini-3.8-flash", "output_tokens": 511, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:37.553060+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0027615, "created_at": "2026-09-17T03:25:37.553Z" }, { "sequence": 3149, "attempt_id": "e0ca6d84-92fe-4e2b-9abb-0d340e8ed0b4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.7018551209475845, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:38.492862+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:25:38.493Z" }, { "sequence": 3150, "attempt_id": "c8f9c8a7-4fce-4a5f-8050-f2a3295aafbd", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00465275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 4.06835527101066, "model_name": "muse-spark-1.3", "output_tokens": 353, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:38.977691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00465275, "created_at": "2026-09-17T03:25:38.977Z" }, { "sequence": 3151, "attempt_id": "2bba5971-eeb0-4119-8e3c-a4796e1ae308", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007126, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 5.094746188959107, "model_name": "gemini-3.1-pro-preview", "output_tokens": 406, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:39.146747+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007126, "created_at": "2026-09-17T03:25:39.146Z" }, { "sequence": 3152, "attempt_id": "22c6ff32-7279-4716-b8f7-ba0a3f37ca38", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 0.8436961839906871, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:39.515750+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:25:39.515Z" }, { "sequence": 3153, "attempt_id": "6fbfc349-61fb-42c8-a8b2-896fbaf554a4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.0242213560268283, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:39.695036+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:25:39.695Z" }, { "sequence": 3154, "attempt_id": "14024fce-682e-4924-8dba-cf258e7bcd4b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0058895, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 4.599513876019046, "model_name": "muse-spark-1.2", "output_tokens": 644, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:40.220570+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0058895, "created_at": "2026-09-17T03:25:40.220Z" }, { "sequence": 3155, "attempt_id": "6a8a6ea7-0693-4830-8574-5142b123cc97", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075855, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 10.20547621708829, "model_name": "muse-spark-1.3", "output_tokens": 1041, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:40.388024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0075855, "created_at": "2026-09-17T03:25:40.388Z" }, { "sequence": 3156, "attempt_id": "3bb6db6e-0dbd-4e37-a600-51268f5b4307", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.8353924880502746, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:40.553555+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003764, "created_at": "2026-09-17T03:25:40.553Z" }, { "sequence": 3157, "attempt_id": "18036ed6-e416-410e-9f09-cf99daf5da13", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006684, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.918843007995747, "model_name": "gpt-5.6-terra", "output_tokens": 154, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:40.732632+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006684, "created_at": "2026-09-17T03:25:40.732Z" }, { "sequence": 3158, "attempt_id": "3eb44f99-1764-46ae-8d59-848880dc674e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0114785, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 10.529710147995502, "model_name": "muse-spark-1.2", "output_tokens": 1957, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:40.898722+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0114785, "created_at": "2026-09-17T03:25:40.899Z" }, { "sequence": 3159, "attempt_id": "abacace9-60e0-495f-b04a-562cebbbaded", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0608785490039736, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:41.793848+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:25:41.794Z" }, { "sequence": 3160, "attempt_id": "c7d1e655-71eb-4318-acc6-0c05c944be05", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038835, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.873310983995907, "model_name": "gemini-3.8-flash", "output_tokens": 803, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:42.184878+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0038835, "created_at": "2026-09-17T03:25:42.185Z" }, { "sequence": 3161, "attempt_id": "01bbe97f-5f7d-49a9-8a6a-ff99df712614", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0097935, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 18.074276293977164, "model_name": "muse-spark-1.3", "output_tokens": 1557, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:42.367130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0097935, "created_at": "2026-09-17T03:25:42.367Z" }, { "sequence": 3162, "attempt_id": "b5bd561e-ccc3-441e-9d49-d83b031dfe10", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005248, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3710415579844266, "model_name": "gpt-5.6-terra", "output_tokens": 39, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:42.527177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005248, "created_at": "2026-09-17T03:25:42.527Z" }, { "sequence": 3163, "attempt_id": "a63e58c0-5e9f-40bd-a5b5-a042ed6e2bf0", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.886167115997523, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:42.706041+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:25:42.706Z" }, { "sequence": 3164, "attempt_id": "9d009d67-214f-41e0-b713-3eca09e2adc3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.384630226995796, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:43.448990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:25:43.449Z" }, { "sequence": 3165, "attempt_id": "3acd3d06-73ef-4e09-b3bf-b8d415f2bb21", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8866891589714214, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:43.634521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:25:43.634Z" }, { "sequence": 3166, "attempt_id": "89ca5b6f-b33b-4a27-a13d-62deefaf07b0", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005854, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.368368493043818, "model_name": "gemini-3.1-pro-preview", "output_tokens": 294, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:43.793657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005854, "created_at": "2026-09-17T03:25:43.793Z" }, { "sequence": 3167, "attempt_id": "d64518d1-bdfb-42e1-b28f-e322963d69db", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00460025, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.992594221024774, "model_name": "muse-spark-1.2", "output_tokens": 338, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:44.866395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00460025, "created_at": "2026-09-17T03:25:44.866Z" }, { "sequence": 3168, "attempt_id": "eec0e4f8-a528-43cc-99b2-d7b77314df90", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0459219799377024, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:45.052619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:25:45.054Z" }, { "sequence": 3169, "attempt_id": "626dca65-9e34-43c8-87db-d5d7fec58a22", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02241725, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 57.25740506500006, "model_name": "muse-spark-1.3", "output_tokens": 4522, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:25:45.214357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02241725, "created_at": "2026-09-17T03:25:45.214Z" }, { "sequence": 3170, "attempt_id": "375c23be-38af-4586-80a7-6340bb980fb2", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001914, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.714066887041554, "model_name": "gemini-3.8-flash", "output_tokens": 283, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:45.389686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001914, "created_at": "2026-09-17T03:25:45.389Z" }, { "sequence": 3171, "attempt_id": "889bf78d-897d-4133-8cd9-73aba2ea80b3", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.607061784947291, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:45.551082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:25:45.551Z" }, { "sequence": 3172, "attempt_id": "d63c9ebf-3981-4b15-bed4-0f59ca058e44", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0042305, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.71798677300103, "model_name": "muse-spark-1.3", "output_tokens": 251, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:45.734481+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0042305, "created_at": "2026-09-17T03:25:45.734Z" }, { "sequence": 3173, "attempt_id": "b4ce7e02-64f5-43db-b94c-ea037f7ef855", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.037947466946207, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:45.995737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:25:45.995Z" }, { "sequence": 3174, "attempt_id": "a239cb3e-8025-4efe-95eb-6ba6b14c3417", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9854189930483699, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:46.167256+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:25:46.167Z" }, { "sequence": 3175, "attempt_id": "a0514384-0c07-4276-82da-3f3a5bbe59f0", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8103464629966766, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:46.539812+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003561, "created_at": "2026-09-17T03:25:46.539Z" }, { "sequence": 3176, "attempt_id": "77418d09-6ebf-44b3-84ba-d83f185eb61e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122025, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.4211211979854852, "model_name": "gemini-3.8-flash", "output_tokens": 98, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:46.811115+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00122025, "created_at": "2026-09-17T03:25:46.811Z" }, { "sequence": 3177, "attempt_id": "305a4904-5a07-4649-94b4-fbfd04227c05", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.1318980430951342, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:47.469758+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:25:47.469Z" }, { "sequence": 3178, "attempt_id": "c406510d-5bdf-40f4-b215-c306a7007c57", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007554, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.58156154199969, "model_name": "gemini-3.1-pro-preview", "output_tokens": 440, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:47.639840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007554, "created_at": "2026-09-17T03:25:47.640Z" }, { "sequence": 3179, "attempt_id": "036888bc-bb48-4f5c-bfc2-959d32c91bfe", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.769518462009728, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:47.937087+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:25:47.937Z" }, { "sequence": 3180, "attempt_id": "84bb932a-9f9b-41d5-b5d7-5b38065520ff", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00427725, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.1054686550050974, "model_name": "muse-spark-1.2", "output_tokens": 262, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:48.106454+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00427725, "created_at": "2026-09-17T03:25:48.106Z" }, { "sequence": 3181, "attempt_id": "6ad4afcf-6e7c-4c6c-9fb0-7583047cf263", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.0285452660173178, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:48.269278+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:25:48.269Z" }, { "sequence": 3182, "attempt_id": "2d7c6741-6e51-435a-89c8-d73548e69514", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003666, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.794336663908325, "model_name": "gemini-3.1-pro-preview", "output_tokens": 116, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:48.440257+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003666, "created_at": "2026-09-17T03:25:48.440Z" }, { "sequence": 3183, "attempt_id": "bcde235d-1832-468c-87a3-c1617f9085f4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00514, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.787017013062723, "model_name": "gpt-5.6-terra", "output_tokens": 29, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:48.621156+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00514, "created_at": "2026-09-17T03:25:48.621Z" }, { "sequence": 3184, "attempt_id": "5fbc939f-d59d-4734-9b85-5b1b465005a3", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.1459185369312763, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:49.252718+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003707, "created_at": "2026-09-17T03:25:49.252Z" }, { "sequence": 3185, "attempt_id": "4b5afd44-aaf2-4393-8e34-dd4c36c24b16", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004851, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.680411819019355, "model_name": "muse-spark-1.3", "output_tokens": 397, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:49.423490+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004851, "created_at": "2026-09-17T03:25:49.423Z" }, { "sequence": 3186, "attempt_id": "4bdb6cda-a302-4cff-a621-695c10222ac4", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00138675, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.6641497239470482, "model_name": "gemini-3.8-flash", "output_tokens": 141, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:49.591194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00138675, "created_at": "2026-09-17T03:25:49.591Z" }, { "sequence": 3187, "attempt_id": "88b25111-029f-4467-81b4-b3090826576e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.6074183869641274, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:25:50.228926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:25:50.229Z" }, { "sequence": 3188, "attempt_id": "065c8000-9b47-4522-b537-6fd057535364", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00415, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.096631825901568, "model_name": "muse-spark-1.2", "output_tokens": 230, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:50.537193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00415, "created_at": "2026-09-17T03:25:50.537Z" }, { "sequence": 3189, "attempt_id": "355d12ab-8b71-4397-acd9-8e9c53e12aff", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005628, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4257578930119053, "model_name": "gpt-5.6-terra", "output_tokens": 66, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:25:50.849564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005628, "created_at": "2026-09-17T03:25:50.849Z" }, { "sequence": 3190, "attempt_id": "5b322603-97fa-4bdd-adf4-57b8880d452a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.0906470140907913, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:25:51.031369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:25:51.031Z" }, { "sequence": 3191, "attempt_id": "938875a3-c0f9-4e0d-af5f-e4eefe92913d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0124935, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 10.73390313400887, "model_name": "muse-spark-1.2", "output_tokens": 2187, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:51.199670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0124935, "created_at": "2026-09-17T03:25:51.199Z" }, { "sequence": 3192, "attempt_id": "697d63fc-7c01-4997-9eb4-79c184622d9c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.6683396289590746, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:25:51.374290+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:25:51.374Z" }, { "sequence": 3193, "attempt_id": "591cbb96-ff91-42e3-b390-32e2a63c0e03", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004099, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.144107955042273, "model_name": "muse-spark-1.3", "output_tokens": 218, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:51.579227+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004099, "created_at": "2026-09-17T03:25:51.579Z" }, { "sequence": 3194, "attempt_id": "90891982-1a60-4e02-8eb9-bbd832bb5648", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.0415956679498777, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:25:51.770334+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003764, "created_at": "2026-09-17T03:25:51.770Z" }, { "sequence": 3195, "attempt_id": "222b5b90-f814-45d4-a2ea-73e67e2560e7", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00219225, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.7054399619810283, "model_name": "gemini-3.8-flash", "output_tokens": 352, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:25:52.104449+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00219225, "created_at": "2026-09-17T03:25:52.104Z" }, { "sequence": 3196, "attempt_id": "ac1ac44a-9f74-49c7-a6ec-ed9e85f08c02", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005708, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.968669519992545, "model_name": "gemini-3.1-pro-preview", "output_tokens": 285, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:52.610121+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005708, "created_at": "2026-09-17T03:25:52.610Z" }, { "sequence": 3197, "attempt_id": "7d0aac84-7b5e-4443-b36e-75b149b57821", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02606, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.012578728958033, "model_name": "gpt-6-astra", "output_tokens": 28, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:25:53.387249+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02606, "created_at": "2026-09-17T03:25:53.387Z" }, { "sequence": 3198, "attempt_id": "8dce9fee-e1f1-4c41-9615-70e96b596eb2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005856, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.303515161969699, "model_name": "gpt-5.6-terra", "output_tokens": 77, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:25:54.074195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005856, "created_at": "2026-09-17T03:25:54.074Z" }, { "sequence": 3199, "attempt_id": "f3c789b4-0376-4b1f-9204-24f697c558e1", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.6875983340432867, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:25:54.256575+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:25:54.256Z" }, { "sequence": 3200, "attempt_id": "b542cfc0-d38f-4a92-8a4f-2c530e9f9966", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00240225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.9353675961028785, "model_name": "gemini-3.8-flash", "output_tokens": 398, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:25:54.716372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00240225, "created_at": "2026-09-17T03:25:54.716Z" }, { "sequence": 3201, "attempt_id": "d0ba4ae7-4729-422c-8d17-bd2b73152226", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008002, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 5.831615620991215, "model_name": "gemini-3.1-pro-preview", "output_tokens": 473, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:25:55.604311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008002, "created_at": "2026-09-17T03:25:55.604Z" }, { "sequence": 3202, "attempt_id": "d956fbc4-b975-4915-91d2-0b294fa8a895", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012344, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.153251739102416, "model_name": "gpt-5.6-sol", "output_tokens": 124, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:25:55.776242+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012344, "created_at": "2026-09-17T03:25:55.776Z" }, { "sequence": 3203, "attempt_id": "911db5a1-0295-4dc2-b3f6-93479cdd502b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00721925, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 4.883434049086645, "model_name": "muse-spark-1.2", "output_tokens": 946, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:25:56.083419+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00721925, "created_at": "2026-09-17T03:25:56.083Z" }, { "sequence": 3204, "attempt_id": "be0a02c6-a798-4394-8501-2c658bbc94e3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.2461368659278378, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:57.022721+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009864, "created_at": "2026-09-17T03:25:57.022Z" }, { "sequence": 3205, "attempt_id": "00c9c990-d51c-43e4-87f6-6a64154cd3c7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.8601522049866617, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:57.207170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000496, "created_at": "2026-09-17T03:25:57.207Z" }, { "sequence": 3206, "attempt_id": "6d7ce494-aaf4-4c55-b431-f26c8b1cc95d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005392, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.4139444479951635, "model_name": "gpt-5.6-terra", "output_tokens": 50, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:57.376684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005392, "created_at": "2026-09-17T03:25:57.376Z" }, { "sequence": 3207, "attempt_id": "458c8ac8-04e6-480c-a95c-548fefdb7828", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.334380952990614, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:57.540862+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02466, "created_at": "2026-09-17T03:25:57.541Z" }, { "sequence": 3208, "attempt_id": "1a3172c7-a767-4a86-a559-cd1c96b65ca7", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000606, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 5.589019121020101, "model_name": "gpt-5.6-luna", "output_tokens": 94, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:25:57.713114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000606, "created_at": "2026-09-17T03:25:57.714Z" }, { "sequence": 3209, "attempt_id": "f48fbf2a-079b-4c41-8e32-fca3768da972", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003582, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.8306983150541782, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:58.538755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003582, "created_at": "2026-09-17T03:25:58.538Z" }, { "sequence": 3210, "attempt_id": "303f5778-a0f9-4b4d-930c-ce8e3d0e7b43", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009074, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.645248732995242, "model_name": "gemini-3.1-pro-preview", "output_tokens": 554, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:25:58.921163+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009074, "created_at": "2026-09-17T03:25:58.921Z" }, { "sequence": 3211, "attempt_id": "36f33a0a-3c15-4d03-9fdd-f67a4c653c7b", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001878, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.7777441299986094, "model_name": "gemini-3.8-flash", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:59.154717+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001878, "created_at": "2026-09-17T03:25:59.154Z" }, { "sequence": 3212, "attempt_id": "57543f8b-6736-4228-bcfe-35efdb83054b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.1296736729564145, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:00.466503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004904, "created_at": "2026-09-17T03:26:00.466Z" }, { "sequence": 3213, "attempt_id": "4bb2bc2d-3158-478f-a454-5e04404efeaf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.417938136961311, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:00.638242+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009752, "created_at": "2026-09-17T03:26:00.638Z" }, { "sequence": 3214, "attempt_id": "f16ada2a-a40c-43e8-9859-9a63ff0dc51d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.041030828957446, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:00.807888+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02438, "created_at": "2026-09-17T03:26:00.809Z" }, { "sequence": 3215, "attempt_id": "33da9a79-dddc-4920-b72c-323a5cfb8a70", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00518275, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.252273533027619, "model_name": "muse-spark-1.2", "output_tokens": 473, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:01.148798+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00518275, "created_at": "2026-09-17T03:26:01.148Z" }, { "sequence": 3216, "attempt_id": "fad68ca7-bafd-490a-ac71-b7bb0c1dcb65", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00554, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.0387306320481, "model_name": "gemini-3.1-pro-preview", "output_tokens": 271, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:01.330642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00554, "created_at": "2026-09-17T03:26:01.330Z" }, { "sequence": 3217, "attempt_id": "55f1c4a3-5399-4640-959a-2f2139876f94", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00472375, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.9439842300489545, "model_name": "muse-spark-1.3", "output_tokens": 365, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:01.657430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00472375, "created_at": "2026-09-17T03:26:01.657Z" }, { "sequence": 3218, "attempt_id": "541f4727-b695-4040-a85f-ac6c64370b01", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.897829961962998, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:02.536414+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004904, "created_at": "2026-09-17T03:26:02.536Z" }, { "sequence": 3219, "attempt_id": "a1cd4c79-9c85-4ac9-b0f0-c5f4ac3c4f4d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.4561163529288024, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:02.787056+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:26:02.787Z" }, { "sequence": 3220, "attempt_id": "3ba71878-648c-4911-a54e-0e023d7cf108", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125625, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.019477301975712, "model_name": "gemini-3.8-flash", "output_tokens": 112, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:02.997005+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00125625, "created_at": "2026-09-17T03:26:02.997Z" }, { "sequence": 3221, "attempt_id": "2f6a01b1-27be-40a6-a13d-510902c76075", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00413075, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.3316422390053049, "model_name": "muse-spark-1.2", "output_tokens": 234, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:03.167904+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00413075, "created_at": "2026-09-17T03:26:03.169Z" }, { "sequence": 3222, "attempt_id": "a37e0c75-5b10-4c00-bc97-6ded71519e7c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0142105, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 23.391514430986717, "model_name": "muse-spark-1.3", "output_tokens": 2591, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:03.340742+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0142105, "created_at": "2026-09-17T03:26:03.341Z" }, { "sequence": 3223, "attempt_id": "e8be17fe-b0f5-4a43-9dba-b755fea92d2b", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003874, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.865652263048105, "model_name": "gemini-3.1-pro-preview", "output_tokens": 137, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:03.673834+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003874, "created_at": "2026-09-17T03:26:03.674Z" }, { "sequence": 3224, "attempt_id": "6f7016a9-87ce-4bc6-bea4-c8ce18150239", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.6496245269663632, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:04.371588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02474, "created_at": "2026-09-17T03:26:04.371Z" }, { "sequence": 3225, "attempt_id": "4c7bb0e0-ea52-442d-af53-300ed4703324", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009896, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.4482279469957575, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:04.543436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009896, "created_at": "2026-09-17T03:26:04.543Z" }, { "sequence": 3226, "attempt_id": "d6cc9d56-6e60-4157-a2cf-f6574ba6d096", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005636, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.7279986820649356, "model_name": "gpt-5.6-terra", "output_tokens": 69, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:04.896235+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005636, "created_at": "2026-09-17T03:26:04.896Z" }, { "sequence": 3227, "attempt_id": "75a8e47c-cb42-42f9-9c47-e378d4f589f3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00396925, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 3.6965691579971462, "model_name": "muse-spark-1.3", "output_tokens": 196, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:05.196985+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00396925, "created_at": "2026-09-17T03:26:05.197Z" }, { "sequence": 3228, "attempt_id": "e4beb3bf-cd90-40f3-b2f5-ad2862a69078", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.8540387339890003, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:05.397754+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000374, "created_at": "2026-09-17T03:26:05.397Z" }, { "sequence": 3229, "attempt_id": "9daefef1-2dea-40d3-81f4-e7e85fb3d339", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0021525, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.8699889859417453, "model_name": "gemini-3.8-flash", "output_tokens": 343, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:05.715511+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0021525, "created_at": "2026-09-17T03:26:05.715Z" }, { "sequence": 3230, "attempt_id": "499a41cd-a50b-4c23-82e8-1919e382f68a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008839, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 12.638847533031367, "model_name": "muse-spark-1.3", "output_tokens": 1313, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:06.884663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008839, "created_at": "2026-09-17T03:26:06.884Z" }, { "sequence": 3231, "attempt_id": "7d4a9199-6bd8-4ecf-b7c3-f18b2d49773e", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.6750179930822924, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:07.058846+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:26:07.059Z" }, { "sequence": 3232, "attempt_id": "5a08b62c-5868-4128-bfe0-ae9897723fb1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.7064166300697252, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:07.273607+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:26:07.273Z" }, { "sequence": 3233, "attempt_id": "967ec27c-e5f7-455e-824f-b779c201ef40", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005696000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 3.9597162500722334, "model_name": "gpt-5.6-luna", "output_tokens": 74, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:07.444471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005696000000000001, "created_at": "2026-09-17T03:26:07.444Z" }, { "sequence": 3234, "attempt_id": "7715390e-d5f5-451f-9a61-b83254def0fe", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.404059205087833, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:07.624193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:26:07.624Z" }, { "sequence": 3235, "attempt_id": "a51267d4-8652-44db-9c24-46283da2657a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1791793290758505, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:08.238378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:26:08.238Z" }, { "sequence": 3236, "attempt_id": "68ddaa19-b167-47f2-8447-23863e6db003", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054975, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 3.2909865470137447, "model_name": "muse-spark-1.2", "output_tokens": 545, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:08.413444+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0054975, "created_at": "2026-09-17T03:26:08.413Z" }, { "sequence": 3237, "attempt_id": "6ee27884-a2c1-46d5-bf59-8466bf82695c", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003495, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.7687453529797494, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:08.599966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003495, "created_at": "2026-09-17T03:26:08.600Z" }, { "sequence": 3238, "attempt_id": "2d9c6c0b-f7b2-46bc-be79-619280ca58e8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00795, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 5.29071589501109, "model_name": "gemini-3.1-pro-preview", "output_tokens": 470, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:08.814632+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00795, "created_at": "2026-09-17T03:26:08.814Z" }, { "sequence": 3239, "attempt_id": "b03045cb-d3b3-4f37-89f5-58ee91a1f238", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016302, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 14.555025930982083, "model_name": "muse-spark-1.2", "output_tokens": 3069, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:26:08.986475+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016302, "created_at": "2026-09-17T03:26:08.986Z" }, { "sequence": 3240, "attempt_id": "7ba3884f-b73a-4594-9861-32898bf21fa5", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00124125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.7772817320656031, "model_name": "gemini-3.8-flash", "output_tokens": 108, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:09.222059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00124125, "created_at": "2026-09-17T03:26:09.222Z" }, { "sequence": 3241, "attempt_id": "2585aad3-d751-4fba-8c50-b92a7f509b15", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00526375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 4.938053313991986, "model_name": "muse-spark-1.3", "output_tokens": 490, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:09.650749+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00526375, "created_at": "2026-09-17T03:26:09.650Z" }, { "sequence": 3242, "attempt_id": "942b08b1-a2bf-40e2-8d6d-e5cdffc08c84", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0805335739860311, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:10.067349+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004902, "created_at": "2026-09-17T03:26:10.067Z" }, { "sequence": 3243, "attempt_id": "f5f1efee-3807-4d13-9b47-fbd7138f62e0", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.6721308049745858, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:10.272370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02437, "created_at": "2026-09-17T03:26:10.272Z" }, { "sequence": 3244, "attempt_id": "52981e80-0c36-4d70-8751-228202a20cbd", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.1723786090733483, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:10.456624+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004902000000000001, "created_at": "2026-09-17T03:26:10.456Z" }, { "sequence": 3245, "attempt_id": "3166d80e-dd82-434f-91dc-da20e97cf024", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.9277315658982843, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:10.697791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009748, "created_at": "2026-09-17T03:26:10.697Z" }, { "sequence": 3246, "attempt_id": "273cab93-8860-4b06-a738-1400629658f5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00465525, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.1411209820071235, "model_name": "muse-spark-1.3", "output_tokens": 358, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:10.939889+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00465525, "created_at": "2026-09-17T03:26:10.940Z" }, { "sequence": 3247, "attempt_id": "e6a6ff4b-adf9-4ae7-b62d-be83a9c9cd3f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.8204549259971827, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:11.112560+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003617, "created_at": "2026-09-17T03:26:11.114Z" }, { "sequence": 3248, "attempt_id": "bd1d7c9d-58f8-41c6-8643-c837eb8d556c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00411125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.5287994410609826, "model_name": "muse-spark-1.2", "output_tokens": 230, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:11.291337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00411125, "created_at": "2026-09-17T03:26:11.291Z" }, { "sequence": 3249, "attempt_id": "61df100f-f10f-477b-a4b3-0691d8d2b34c", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003598, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.066824064939283, "model_name": "gemini-3.1-pro-preview", "output_tokens": 114, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:11.467689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003598, "created_at": "2026-09-17T03:26:11.467Z" }, { "sequence": 3250, "attempt_id": "bf81282e-3691-4740-bd91-d440a17bf83e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001323, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.4345576650230214, "model_name": "gemini-3.8-flash", "output_tokens": 130, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:11.632391+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001323, "created_at": "2026-09-17T03:26:11.632Z" }, { "sequence": 3251, "attempt_id": "3e2d86f7-e0a5-4134-816d-29354fe4cce9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0604969849810004, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:12.173421+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:26:12.173Z" }, { "sequence": 3252, "attempt_id": "b02d19d7-6903-4c52-af13-c2199c91f9fd", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.4975779470987618, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:12.365195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:26:12.365Z" }, { "sequence": 3253, "attempt_id": "4047df4b-6a4a-4827-947e-cea3d9e95e6a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1660281899385154, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:12.628522+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:26:12.628Z" }, { "sequence": 3254, "attempt_id": "8e087079-7a9c-4a4f-98e4-d09f485b2a51", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00432925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 2.4141277230810374, "model_name": "muse-spark-1.2", "output_tokens": 281, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:13.040965+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00432925, "created_at": "2026-09-17T03:26:13.041Z" }, { "sequence": 3255, "attempt_id": "6db2263a-7279-4ee4-9cad-e2850c6afa89", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.5762504839804024, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:13.227140+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:26:13.227Z" }, { "sequence": 3256, "attempt_id": "fb5788d9-936c-4a6f-848f-047b5075b5a6", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.8971667269943282, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:13.400105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00035009999999999994, "created_at": "2026-09-17T03:26:13.400Z" }, { "sequence": 3257, "attempt_id": "4b1756b6-8738-429d-bcf2-8183c7181fb6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00441, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.1057423130841926, "model_name": "muse-spark-1.3", "output_tokens": 300, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:13.572136+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00441, "created_at": "2026-09-17T03:26:13.572Z" }, { "sequence": 3258, "attempt_id": "e944a642-8df2-4f04-bd24-9034da207964", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00403125, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.2952642209129408, "model_name": "muse-spark-1.2", "output_tokens": 210, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:14.095400+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00403125, "created_at": "2026-09-17T03:26:14.095Z" }, { "sequence": 3259, "attempt_id": "98a6858e-3c3b-4be5-8f90-928a3a82184a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01285475, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 23.464460386079736, "model_name": "muse-spark-1.3", "output_tokens": 2272, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:14.496155+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01285475, "created_at": "2026-09-17T03:26:14.496Z" }, { "sequence": 3260, "attempt_id": "1e84502e-a1d9-4ad2-ab0e-96808ec57d1a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.011452489066869, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:14.669071+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:26:14.669Z" }, { "sequence": 3261, "attempt_id": "4ec7b666-c467-4791-8af8-b5754aeb0532", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.337795335915871, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:14.841905+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:26:14.842Z" }, { "sequence": 3262, "attempt_id": "5c76120c-3427-40d0-b842-a009958a81c1", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005852, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 5.428437864989974, "model_name": "gemini-3.1-pro-preview", "output_tokens": 302, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:15.023230+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005852, "created_at": "2026-09-17T03:26:15.023Z" }, { "sequence": 3263, "attempt_id": "9ce999c8-e6ba-447b-9000-3cd9fab3bb40", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001239, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.0662195440381765, "model_name": "gemini-3.8-flash", "output_tokens": 107, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:15.199877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001239, "created_at": "2026-09-17T03:26:15.200Z" }, { "sequence": 3264, "attempt_id": "fe25cb9c-8a0c-4ce5-beab-30ea5b927e34", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.9578714109957218, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:15.364412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:26:15.364Z" }, { "sequence": 3265, "attempt_id": "c2844aa4-1ae4-4a7e-ba38-ccc832311ba4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004178, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.6916412389837205, "model_name": "gemini-3.1-pro-preview", "output_tokens": 162, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:15.541160+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004178, "created_at": "2026-09-17T03:26:15.541Z" }, { "sequence": 3266, "attempt_id": "8c917e3f-70b8-458d-9e31-ea1f25087f00", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8336727099958807, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:15.726994+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:26:15.727Z" }, { "sequence": 3267, "attempt_id": "1f11a401-b18e-4e4a-985a-aba8dedf7728", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.296705287997611, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:16.046273+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:26:16.046Z" }, { "sequence": 3268, "attempt_id": "8830bdff-5658-435a-8e99-532e09aaea61", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00453275, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.540435035014525, "model_name": "muse-spark-1.3", "output_tokens": 328, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:16.223140+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00453275, "created_at": "2026-09-17T03:26:16.224Z" }, { "sequence": 3269, "attempt_id": "f55d077a-b213-4c3e-ac27-dd9da6808f1a", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001329, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.5243714089738205, "model_name": "gemini-3.8-flash", "output_tokens": 127, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:16.402919+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001329, "created_at": "2026-09-17T03:26:16.403Z" }, { "sequence": 3270, "attempt_id": "d86b1420-0ae7-48cb-a340-adf9c871a7f5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.048786905943416, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:16.590211+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009744, "created_at": "2026-09-17T03:26:16.590Z" }, { "sequence": 3271, "attempt_id": "812ac043-80ac-49c7-8253-498c507bbc06", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 0.7772563899634406, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:16.763785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004900000000000001, "created_at": "2026-09-17T03:26:16.764Z" }, { "sequence": 3272, "attempt_id": "b39a8f23-f53b-4834-ad0c-3888414438b3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.13198275398463, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:16.938843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0049, "created_at": "2026-09-17T03:26:16.940Z" }, { "sequence": 3273, "attempt_id": "6d2d765b-7bbc-443e-b1bd-20a22119dad0", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.8720185570418835, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:17.452422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.000362, "created_at": "2026-09-17T03:26:17.452Z" }, { "sequence": 3274, "attempt_id": "baecb0de-9ff0-4668-a7ea-617e8ca0f269", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.011195863946341, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:17.628391+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02436, "created_at": "2026-09-17T03:26:17.630Z" }, { "sequence": 3275, "attempt_id": "1f39776f-126a-461d-b543-8da8639e81cc", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003486, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.3395706430310383, "model_name": "gemini-3.1-pro-preview", "output_tokens": 101, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:17.794207+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003486, "created_at": "2026-09-17T03:26:17.794Z" }, { "sequence": 3276, "attempt_id": "debb5af7-2127-4c76-ae80-e80e5c477aba", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00144, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3073555279988796, "model_name": "gemini-3.8-flash", "output_tokens": 161, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:17.985678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00144, "created_at": "2026-09-17T03:26:17.985Z" }, { "sequence": 3277, "attempt_id": "360a9821-2e34-4113-a387-2a94a6fce4f7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00452375, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.848124483949505, "model_name": "muse-spark-1.2", "output_tokens": 320, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:18.161587+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00452375, "created_at": "2026-09-17T03:26:18.161Z" }, { "sequence": 3278, "attempt_id": "4c041bda-13a9-4739-8c0c-d0df9ba48e2b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.0958944840822369, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:18.724624+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:26:18.724Z" }, { "sequence": 3279, "attempt_id": "12c29aa8-fc44-499a-b64d-87171672d470", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003466, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.8439926500432193, "model_name": "gemini-3.1-pro-preview", "output_tokens": 103, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:19.067460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003466, "created_at": "2026-09-17T03:26:19.067Z" }, { "sequence": 3280, "attempt_id": "1ec99d21-4de6-49a2-b6aa-e0e2b2a2702f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7352792739402503, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:19.635472+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:26:19.635Z" }, { "sequence": 3281, "attempt_id": "69ac9cd4-228f-479e-a85b-c88c3d824006", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0057, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.8561351000098512, "model_name": "gpt-5.6-terra", "output_tokens": 75, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:19.836487+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0057, "created_at": "2026-09-17T03:26:19.836Z" }, { "sequence": 3282, "attempt_id": "0fe386bc-065b-4749-92ef-abe4dc3d7a49", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00482525, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.746577135985717, "model_name": "muse-spark-1.3", "output_tokens": 398, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:20.510708+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00482525, "created_at": "2026-09-17T03:26:20.510Z" }, { "sequence": 3283, "attempt_id": "e1bf4864-b2ee-46e7-97d1-259a5590ad2f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00259125, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.557222079951316, "model_name": "gemini-3.8-flash", "output_tokens": 461, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:20.890644+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00259125, "created_at": "2026-09-17T03:26:20.890Z" }, { "sequence": 3284, "attempt_id": "5a3b33a7-02b4-4051-adbe-804999907aa0", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.7697309349896386, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:21.065517+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:26:21.067Z" }, { "sequence": 3285, "attempt_id": "1d00c65c-1475-4817-9ef0-cfa7da17152e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00617675, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 4.547867043991573, "model_name": "muse-spark-1.2", "output_tokens": 716, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:21.487051+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00617675, "created_at": "2026-09-17T03:26:21.487Z" }, { "sequence": 3286, "attempt_id": "c8ea5720-d60d-4b19-9275-89f98f5d12f8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005748, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 4.131165080005303, "model_name": "gpt-5.6-luna", "output_tokens": 79, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:22.117185+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005748, "created_at": "2026-09-17T03:26:22.117Z" }, { "sequence": 3287, "attempt_id": "5c840ca5-7e34-4e9b-add4-1d2ec4a94fdd", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02736, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.592380615998991, "model_name": "gpt-6-astra", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:22.602654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02736, "created_at": "2026-09-17T03:26:22.602Z" }, { "sequence": 3288, "attempt_id": "ea386ae5-8b67-4a52-9666-fdd016de42d6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011824, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.1763523479457945, "model_name": "gpt-5.6-sol", "output_tokens": 98, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:22.875658+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011824, "created_at": "2026-09-17T03:26:22.875Z" }, { "sequence": 3289, "attempt_id": "3cd68f3e-b9bb-4e03-97cf-610f06377bd6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005988, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.1796841549221426, "model_name": "gpt-5.6-terra", "output_tokens": 88, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:23.245534+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005988, "created_at": "2026-09-17T03:26:23.245Z" }, { "sequence": 3290, "attempt_id": "3b6b6e7a-d348-4d1f-85ec-df981ca5a607", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00604925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 3.762788377935067, "model_name": "muse-spark-1.2", "output_tokens": 676, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:23.574388+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00604925, "created_at": "2026-09-17T03:26:23.574Z" }, { "sequence": 3291, "attempt_id": "41eb620c-cb9b-4dea-bb3b-f64556fe6130", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-016", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7968223060015589, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:26:23.749432+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:26:23.751Z" }, { "sequence": 3292, "attempt_id": "626df213-ed2b-43aa-8391-ff713d37369d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002031, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.6003174340585247, "model_name": "gemini-3.8-flash", "output_tokens": 299, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:24.904204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002031, "created_at": "2026-09-17T03:26:24.904Z" }, { "sequence": 3293, "attempt_id": "a8d295c0-0126-42b7-a93f-4bd31847b949", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006192000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.404680470936, "model_name": "gpt-5.6-luna", "output_tokens": 105, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:25.645396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006192000000000001, "created_at": "2026-09-17T03:26:25.645Z" }, { "sequence": 3294, "attempt_id": "809a1042-c970-41d0-88fc-db55510d93e0", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7981252560857683, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:25.837471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:26:25.839Z" }, { "sequence": 3295, "attempt_id": "efb9129c-1885-4557-9354-3495bc3fefce", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.6452339300885797, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:26.395028+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:26:26.395Z" }, { "sequence": 3296, "attempt_id": "3569597b-cef1-4148-acfb-3df3df36520e", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00686, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 8.293730500969104, "model_name": "gemini-3.1-pro-preview", "output_tokens": 380, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:26.572674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00686, "created_at": "2026-09-17T03:26:26.574Z" }, { "sequence": 3297, "attempt_id": "014a2175-1477-42ac-88c6-74f6462e5948", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.4974683240288869, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:26.761468+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:26:26.761Z" }, { "sequence": 3298, "attempt_id": "02b42afa-57c5-448d-b916-076599b55dc1", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9146756189875305, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:26.930182+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:26:26.930Z" }, { "sequence": 3299, "attempt_id": "16e59de9-7021-4680-998a-f899f59e5bc4", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0062405, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 8.159184337011538, "model_name": "muse-spark-1.3", "output_tokens": 721, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:27.413445+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0062405, "created_at": "2026-09-17T03:26:27.413Z" }, { "sequence": 3300, "attempt_id": "3bbb22e3-3966-4ba1-8705-cfbaf2e0a1eb", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7489059220533818, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:27.592828+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:26:27.593Z" }, { "sequence": 3301, "attempt_id": "c02e91be-f770-4a91-88b8-f6ce5e6b7142", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008918, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.814857040997595, "model_name": "gemini-3.1-pro-preview", "output_tokens": 541, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:27.774798+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008918, "created_at": "2026-09-17T03:26:27.774Z" }, { "sequence": 3302, "attempt_id": "c43b3ea5-89cf-4be4-a1a3-07259561eff9", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001188, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.1206810439471155, "model_name": "gemini-3.8-flash", "output_tokens": 90, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:27.956053+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001188, "created_at": "2026-09-17T03:26:27.957Z" }, { "sequence": 3303, "attempt_id": "be702ea4-617c-4f64-8003-38df64b7f5e5", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004698, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 12.856390536995605, "model_name": "muse-spark-1.3", "output_tokens": 361, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:28.131926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004698, "created_at": "2026-09-17T03:26:28.132Z" }, { "sequence": 3304, "attempt_id": "7eb4626b-7163-474c-9d10-7a2bc72ab16d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00404825, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.2448150649433956, "model_name": "muse-spark-1.2", "output_tokens": 209, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:28.354987+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00404825, "created_at": "2026-09-17T03:26:28.355Z" }, { "sequence": 3305, "attempt_id": "2181c2da-ac1c-48eb-afdc-d963b427243a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005184, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.9955424609361216, "model_name": "gpt-5.6-terra", "output_tokens": 32, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:28.951945+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005184, "created_at": "2026-09-17T03:26:28.952Z" }, { "sequence": 3306, "attempt_id": "f14e4acf-f21e-458f-847c-bd7a71c3009e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00745775, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 5.708480152999982, "model_name": "muse-spark-1.2", "output_tokens": 988, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:29.142405+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00745775, "created_at": "2026-09-17T03:26:29.142Z" }, { "sequence": 3307, "attempt_id": "8d16f0b4-32fe-4a49-9b9f-9430a060b361", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.22551342996303, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:29.328217+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:26:29.328Z" }, { "sequence": 3308, "attempt_id": "18663a5c-5f04-4b7d-906b-a061ed7529b6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7259342690231279, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:29.509695+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:26:29.509Z" }, { "sequence": 3309, "attempt_id": "dcc2f498-0175-42da-a61f-4fa4866d134c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00404825, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.576402386999689, "model_name": "muse-spark-1.3", "output_tokens": 209, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:29.687154+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00404825, "created_at": "2026-09-17T03:26:29.687Z" }, { "sequence": 3310, "attempt_id": "93684435-5ff6-4ac4-87f3-97960bff4f40", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00160875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.3576240780530497, "model_name": "gemini-3.8-flash", "output_tokens": 199, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:29.888626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00160875, "created_at": "2026-09-17T03:26:29.888Z" }, { "sequence": 3311, "attempt_id": "a5a37882-afd3-4fba-a687-a7baf847cb11", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7818345410050824, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:30.077120+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:26:30.078Z" }, { "sequence": 3312, "attempt_id": "a212f575-3c44-4f8b-b4fb-bf21fd6cc03d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00408, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 4.949552930891514, "model_name": "gemini-3.1-pro-preview", "output_tokens": 151, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:30.966500+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00408, "created_at": "2026-09-17T03:26:30.966Z" }, { "sequence": 3313, "attempt_id": "0ab5346d-6ec1-46c8-88c1-2ab2685bfbc5", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.086420360021293, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:31.773886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:26:31.774Z" }, { "sequence": 3314, "attempt_id": "fc443393-9c6b-4948-b970-91d8e04aa6c8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.677944949013181, "model_name": "gemini-3.1-pro-preview", "output_tokens": 125, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:31.997470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0038, "created_at": "2026-09-17T03:26:31.997Z" }, { "sequence": 3315, "attempt_id": "8c8afb8a-f46b-4cda-a84d-d870f260f886", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005988, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.997497473959811, "model_name": "gpt-5.6-terra", "output_tokens": 88, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:32.183696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005988, "created_at": "2026-09-17T03:26:32.183Z" }, { "sequence": 3316, "attempt_id": "0f0417aa-f0b3-44df-bb07-f58a74cba764", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011424, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.4886579889571294, "model_name": "gpt-5.6-sol", "output_tokens": 78, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:32.360105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011424, "created_at": "2026-09-17T03:26:32.360Z" }, { "sequence": 3317, "attempt_id": "5a267580-1689-4e7e-858c-6cc791d8fffc", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005688, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 4.489857338019647, "model_name": "gpt-5.6-luna", "output_tokens": 74, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:32.622103+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005688, "created_at": "2026-09-17T03:26:32.622Z" }, { "sequence": 3318, "attempt_id": "db65b62f-db96-4095-bc02-c32ad0c5ebfc", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7594605199992657, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:26:32.943464+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:26:32.943Z" }, { "sequence": 3319, "attempt_id": "d7e6843b-d4db-4e57-a81a-dc7c310315a9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00792525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.42738997307606, "model_name": "muse-spark-1.3", "output_tokens": 1098, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:33.476546+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00792525, "created_at": "2026-09-17T03:26:33.476Z" }, { "sequence": 3320, "attempt_id": "f9dc0840-bbba-465c-9c39-ff061da9a690", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00234225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.8541606669314206, "model_name": "gemini-3.8-flash", "output_tokens": 382, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "google", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:26:33.805924+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00234225, "created_at": "2026-09-17T03:26:33.806Z" }, { "sequence": 3321, "attempt_id": "a2b49e5b-81e4-492a-a9bf-8dbc2b7f5f5f", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.6902852200437337, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:34.499766+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:26:34.499Z" }, { "sequence": 3322, "attempt_id": "cfb9b591-2e6e-45c8-80b1-3b217341f468", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010144, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.6039394800318405, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:34.722112+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010144, "created_at": "2026-09-17T03:26:34.722Z" }, { "sequence": 3323, "attempt_id": "0875273c-74d8-47d1-91d7-7e01f327b180", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006636, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.989121827995405, "model_name": "gpt-5.6-luna", "output_tokens": 142, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:35.242592+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006636, "created_at": "2026-09-17T03:26:35.242Z" }, { "sequence": 3324, "attempt_id": "edfd6836-191d-41f7-877e-a0568b832858", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00698425, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.764425178989768, "model_name": "muse-spark-1.2", "output_tokens": 896, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:35.446299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00698425, "created_at": "2026-09-17T03:26:35.446Z" }, { "sequence": 3325, "attempt_id": "211eb077-009f-4c92-bc39-33e1ca3389ba", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00578575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.272609345964156, "model_name": "muse-spark-1.3", "output_tokens": 614, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:35.634536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00578575, "created_at": "2026-09-17T03:26:35.634Z" }, { "sequence": 3326, "attempt_id": "595a2b4b-0fb2-425b-8eb0-bbcf70861a4b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006024, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.0716010679025203, "model_name": "gpt-5.6-terra", "output_tokens": 91, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:35.812471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006024, "created_at": "2026-09-17T03:26:35.812Z" }, { "sequence": 3327, "attempt_id": "24307349-25a3-4c10-9a33-043edc2cd2c9", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-015", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8088879610877484, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:36.228140+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:26:36.228Z" }, { "sequence": 3328, "attempt_id": "c85dca68-af86-4971-85d0-65fb5b23ae95", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001911, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.6479076870018616, "model_name": "gemini-3.8-flash", "output_tokens": 267, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:36.544404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001911, "created_at": "2026-09-17T03:26:36.544Z" }, { "sequence": 3329, "attempt_id": "3a2771f3-ea51-4ed1-a492-a647399f89f4", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.5505200320621952, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:37.539649+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:26:37.539Z" }, { "sequence": 3330, "attempt_id": "2604ef45-f9e0-435a-85e4-7f56bf3de293", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9789502540370449, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:37.719929+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:26:37.720Z" }, { "sequence": 3331, "attempt_id": "80f2f100-89d8-441d-b741-89f74de89f50", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3471710970625281, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:37.895727+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:26:37.897Z" }, { "sequence": 3332, "attempt_id": "27e54b27-330e-42bb-998a-7220293e8a18", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006108000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.31092414597515, "model_name": "gpt-5.6-luna", "output_tokens": 98, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:38.292130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006108000000000001, "created_at": "2026-09-17T03:26:38.292Z" }, { "sequence": 3333, "attempt_id": "59e5b348-e566-4dc5-a579-e1106628d0ac", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.7423271039733663, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:39.212543+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:26:39.212Z" }, { "sequence": 3334, "attempt_id": "e09575ec-f255-4013-943c-57ef3f96fa24", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00130275, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.1589400880038738, "model_name": "gemini-3.8-flash", "output_tokens": 120, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:39.391078+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00130275, "created_at": "2026-09-17T03:26:39.391Z" }, { "sequence": 3335, "attempt_id": "f5b3aec0-3da0-4598-9ff2-50ac4fa7b2cc", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012818, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.414530080044642, "model_name": "gemini-3.1-pro-preview", "output_tokens": 866, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "google", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:26:39.567202+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012818, "created_at": "2026-09-17T03:26:39.567Z" }, { "sequence": 3336, "attempt_id": "075a7c4b-c4e9-44b5-a369-8958e0b920a3", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00738125, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 7.723235183977522, "model_name": "muse-spark-1.3", "output_tokens": 970, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:40.083666+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00738125, "created_at": "2026-09-17T03:26:40.083Z" }, { "sequence": 3337, "attempt_id": "ce2461d4-b8a6-4187-b6c3-2017e11ec552", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00656525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 4.336979693965986, "model_name": "muse-spark-1.2", "output_tokens": 778, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:40.263066+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00656525, "created_at": "2026-09-17T03:26:40.263Z" }, { "sequence": 3338, "attempt_id": "037ce7a3-e6dc-4cc7-8e2d-514dd1fca4bc", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.8202421810710803, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:40.540453+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:26:40.540Z" }, { "sequence": 3339, "attempt_id": "d223c1b1-8f9d-41e9-a9e9-9480ba813949", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0042475, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.37120270205196, "model_name": "muse-spark-1.2", "output_tokens": 255, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:40.938707+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0042475, "created_at": "2026-09-17T03:26:40.938Z" }, { "sequence": 3340, "attempt_id": "b0ea7a73-aa07-41d5-b31c-2667323a2371", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.5179676009574905, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:41.273024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0244, "created_at": "2026-09-17T03:26:41.273Z" }, { "sequence": 3341, "attempt_id": "c63bca86-d64a-4b78-92f9-5a53090baa0d", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9825287599815056, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:41.452154+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004908, "created_at": "2026-09-17T03:26:41.452Z" }, { "sequence": 3342, "attempt_id": "ed7df855-d0b1-436a-a7db-da17f0ebb0b3", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.038354926975444, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:41.622378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00976, "created_at": "2026-09-17T03:26:41.622Z" }, { "sequence": 3343, "attempt_id": "a87bb156-43ad-4caf-b462-026b9a377833", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0596769890980795, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:41.817647+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004908, "created_at": "2026-09-17T03:26:41.817Z" }, { "sequence": 3344, "attempt_id": "a829890f-63da-4deb-958a-6e2a94fc09de", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003714, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.7559834440471604, "model_name": "gemini-3.1-pro-preview", "output_tokens": 120, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:41.990159+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003714, "created_at": "2026-09-17T03:26:41.990Z" }, { "sequence": 3345, "attempt_id": "4dff1fc4-f06b-459c-8093-60927c43dba1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00668, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 6.337026598979719, "model_name": "muse-spark-1.3", "output_tokens": 805, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:42.170804+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00668, "created_at": "2026-09-17T03:26:42.171Z" }, { "sequence": 3346, "attempt_id": "19281de8-103c-433b-9be0-3f2664b88a0a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00437925, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.6597149239387363, "model_name": "muse-spark-1.3", "output_tokens": 286, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:42.352152+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00437925, "created_at": "2026-09-17T03:26:42.352Z" }, { "sequence": 3347, "attempt_id": "188a0b50-996b-4b94-b0cd-9f3fbc17047c", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.8028984219999984, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:42.615114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003626, "created_at": "2026-09-17T03:26:42.615Z" }, { "sequence": 3348, "attempt_id": "2677a7c7-20c8-4625-9935-15f5c1c09620", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007898, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.6101007029647, "model_name": "gemini-3.1-pro-preview", "output_tokens": 456, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:43.299317+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007898, "created_at": "2026-09-17T03:26:43.299Z" }, { "sequence": 3349, "attempt_id": "5cde2d15-7288-4463-9056-9f08b9866bce", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00387825, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.2301527929957956, "model_name": "muse-spark-1.2", "output_tokens": 174, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:43.477601+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00387825, "created_at": "2026-09-17T03:26:43.477Z" }, { "sequence": 3350, "attempt_id": "9b3e3bc7-e784-4c66-b30b-5d775c6cbe61", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.135414042044431, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:43.668450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004946, "created_at": "2026-09-17T03:26:43.668Z" }, { "sequence": 3351, "attempt_id": "61953ecd-938f-4aaa-9789-f8015d767052", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.2828670010203496, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:43.847151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:26:43.847Z" }, { "sequence": 3352, "attempt_id": "d5e9cc69-1d01-47c9-b087-efef7a6d46e1", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00144525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.2455130990128964, "model_name": "gemini-3.8-flash", "output_tokens": 162, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:44.017273+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00144525, "created_at": "2026-09-17T03:26:44.017Z" }, { "sequence": 3353, "attempt_id": "e289d4ee-873c-47f1-8cf6-f76e9f66a258", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.7095739740179852, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:44.210414+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:26:44.211Z" }, { "sequence": 3354, "attempt_id": "7013bcf3-9ec7-4204-8dab-13fb9943dbac", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003458, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.2465716230217367, "model_name": "gemini-3.1-pro-preview", "output_tokens": 102, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:44.389927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003458, "created_at": "2026-09-17T03:26:44.390Z" }, { "sequence": 3355, "attempt_id": "7cc01263-7708-49a5-87f5-5689797722cf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.6565872950013727, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:44.566737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:26:44.566Z" }, { "sequence": 3356, "attempt_id": "f30b2240-a18d-4411-a42b-bb3ecc71ebf6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004418, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.577896742033772, "model_name": "muse-spark-1.3", "output_tokens": 301, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:44.751831+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004418, "created_at": "2026-09-17T03:26:44.751Z" }, { "sequence": 3357, "attempt_id": "411a6406-3850-4bde-8518-d51daf27c33b", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.86955233104527, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:44.932816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003683, "created_at": "2026-09-17T03:26:44.933Z" }, { "sequence": 3358, "attempt_id": "9d139100-47be-4758-9926-0c4a6da3b683", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0018495, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.6918364909943193, "model_name": "gemini-3.8-flash", "output_tokens": 266, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:45.360614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0018495, "created_at": "2026-09-17T03:26:45.360Z" }, { "sequence": 3359, "attempt_id": "e3ed8def-3196-40d0-a124-2b1f9797b5c8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0439608671003953, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:45.611010+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:26:45.611Z" }, { "sequence": 3360, "attempt_id": "490f8690-b44f-4dd6-823b-fc4a851e037b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005224, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.346108243917115, "model_name": "gpt-5.6-terra", "output_tokens": 37, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:46.273542+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005224, "created_at": "2026-09-17T03:26:46.273Z" }, { "sequence": 3361, "attempt_id": "c7a9ebc3-c9d2-4fb0-a48f-4ad5475cc59d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.2042617690749466, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:46.594503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:26:46.594Z" }, { "sequence": 3362, "attempt_id": "f3025d2c-48b3-4bdd-abb8-64ff34e9df70", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8560610719723627, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:46.773420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:26:46.773Z" }, { "sequence": 3363, "attempt_id": "2eee666b-4e32-4493-baa6-e846d162b44c", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00178275, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.6193369969259948, "model_name": "gemini-3.8-flash", "output_tokens": 248, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:47.160757+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00178275, "created_at": "2026-09-17T03:26:47.160Z" }, { "sequence": 3364, "attempt_id": "c783aabe-6a21-4495-8783-e11d12690806", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005432, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.408427205053158, "model_name": "muse-spark-1.2", "output_tokens": 534, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:47.619184+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005432, "created_at": "2026-09-17T03:26:47.619Z" }, { "sequence": 3365, "attempt_id": "c711698f-05a9-4276-9724-125979262cd2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017152, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 15.383360221982002, "model_name": "muse-spark-1.2", "output_tokens": 3269, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:26:47.915623+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017152, "created_at": "2026-09-17T03:26:47.915Z" }, { "sequence": 3366, "attempt_id": "cebedca6-15c0-4fae-b67f-0afcb1835533", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006304, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.500501325004734, "model_name": "gemini-3.1-pro-preview", "output_tokens": 336, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:48.094698+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006304, "created_at": "2026-09-17T03:26:48.096Z" }, { "sequence": 3367, "attempt_id": "4a8e2397-138a-47bf-bb58-74229e7cab1f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 3.1527634139638394, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:48.278249+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:26:48.278Z" }, { "sequence": 3368, "attempt_id": "73db7a78-dade-4af4-b213-77fec9111a9f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00440475, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.7003626050427556, "model_name": "muse-spark-1.2", "output_tokens": 292, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:48.474077+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00440475, "created_at": "2026-09-17T03:26:48.474Z" }, { "sequence": 3369, "attempt_id": "a382a1bb-9da8-45e8-b866-132b127cd634", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7456864599371329, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:26:49.398720+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00039139999999999997, "created_at": "2026-09-17T03:26:49.398Z" }, { "sequence": 3370, "attempt_id": "e8a9269a-730e-4a71-a0a1-d5881def1da2", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00639, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.546405446017161, "model_name": "gemini-3.1-pro-preview", "output_tokens": 343, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:49.655498+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00639, "created_at": "2026-09-17T03:26:49.655Z" }, { "sequence": 3371, "attempt_id": "370acd3e-f6bf-405f-8df0-fc9a32875da9", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004409, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.2296659239800647, "model_name": "muse-spark-1.3", "output_tokens": 293, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:49.833938+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004409, "created_at": "2026-09-17T03:26:49.834Z" }, { "sequence": 3372, "attempt_id": "0b21d260-f41a-450f-bbe2-d845b9b5f776", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006384000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.8707513430854306, "model_name": "gpt-5.6-luna", "output_tokens": 121, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:26:50.011808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006384000000000001, "created_at": "2026-09-17T03:26:50.013Z" }, { "sequence": 3373, "attempt_id": "4507ecdf-b917-4a2e-a816-dd274c897770", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00582, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.2940786159597337, "model_name": "gpt-5.6-terra", "output_tokens": 74, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:26:50.193641+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00582, "created_at": "2026-09-17T03:26:50.193Z" }, { "sequence": 3374, "attempt_id": "5e659715-14c8-44c7-b1ea-bbc95a757c23", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00532575, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 6.100056966068223, "model_name": "muse-spark-1.3", "output_tokens": 509, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:50.384743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00532575, "created_at": "2026-09-17T03:26:50.384Z" }, { "sequence": 3375, "attempt_id": "18bafaf7-d6a3-4e0f-a41f-7b878a16db3d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.5953790970379487, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:26:50.562674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02536, "created_at": "2026-09-17T03:26:50.562Z" }, { "sequence": 3376, "attempt_id": "c3694ca9-9f44-4891-a77c-6ecc33640f88", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011924, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.8844632350374013, "model_name": "gpt-5.6-sol", "output_tokens": 103, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:26:51.228820+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011924, "created_at": "2026-09-17T03:26:51.229Z" }, { "sequence": 3377, "attempt_id": "6b4ddf6d-b14d-4c78-b496-e9dfb63ef358", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.7704524119617417, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:51.426297+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004946, "created_at": "2026-09-17T03:26:51.426Z" }, { "sequence": 3378, "attempt_id": "ae3273eb-20e5-45eb-af7d-3dd74b6658e4", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.171891757985577, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:51.603961+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009836, "created_at": "2026-09-17T03:26:51.604Z" }, { "sequence": 3379, "attempt_id": "746c99c2-bfd9-4f7a-b1fb-da4788957eec", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005354, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.289031014079228, "model_name": "gpt-5.6-terra", "output_tokens": 48, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:51.788179+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005354, "created_at": "2026-09-17T03:26:51.788Z" }, { "sequence": 3380, "attempt_id": "d6190e2a-ad43-4c00-82b4-c94d3b8a9850", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.927152162999846, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:51.981262+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02459, "created_at": "2026-09-17T03:26:51.982Z" }, { "sequence": 3381, "attempt_id": "65c4150f-3b69-4271-92be-c34e96c14223", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.5341877520550042, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:52.316850+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003683, "created_at": "2026-09-17T03:26:52.317Z" }, { "sequence": 3382, "attempt_id": "c1ad07dd-dddc-4dba-bbb1-7bd3bab4ed33", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 4.032184044015594, "model_name": "gemini-3.1-pro-preview", "output_tokens": 282, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:26:52.497324+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T03:26:52.498Z" }, { "sequence": 3383, "attempt_id": "3748f5ce-576f-4948-8aa9-b74a5f47ee8c", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00150225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 4.436489715008065, "model_name": "gemini-3.8-flash", "output_tokens": 158, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:26:52.897435+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00150225, "created_at": "2026-09-17T03:26:52.897Z" }, { "sequence": 3384, "attempt_id": "99bb06a7-d93a-43af-a108-871534e73671", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0480688950046897, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:53.545714+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:26:53.545Z" }, { "sequence": 3385, "attempt_id": "888b55a5-0fe6-4361-a933-155dca012050", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006527, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 3.7681956259766594, "model_name": "muse-spark-1.2", "output_tokens": 769, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:26:53.728441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006527, "created_at": "2026-09-17T03:26:53.728Z" }, { "sequence": 3386, "attempt_id": "3566c82a-2fa3-4b8d-87ba-1393364c485c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0446943810675293, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:53.900785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:26:53.900Z" }, { "sequence": 3387, "attempt_id": "2bfe2dd8-6724-46fd-8917-67c0759d606c", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.605053577106446, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:54.096154+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:26:54.096Z" }, { "sequence": 3388, "attempt_id": "8a498185-3f4e-4927-a646-b70c59d0731c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.138207909069024, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:54.276642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:26:54.276Z" }, { "sequence": 3389, "attempt_id": "da8fe6c1-c03e-4f1f-8a70-68b2a8b97ce5", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00254325, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.715864907950163, "model_name": "gemini-3.8-flash", "output_tokens": 451, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:54.447002+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00254325, "created_at": "2026-09-17T03:26:54.447Z" }, { "sequence": 3390, "attempt_id": "97dcc648-c912-424d-a212-835a4c088296", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8982682109344751, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:54.994762+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036859999999999996, "created_at": "2026-09-17T03:26:54.994Z" }, { "sequence": 3391, "attempt_id": "cca8c25c-ba3b-461c-bbc3-9935c889ddcb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00592075, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.2572774291038513, "model_name": "muse-spark-1.2", "output_tokens": 649, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:55.238857+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00592075, "created_at": "2026-09-17T03:26:55.239Z" }, { "sequence": 3392, "attempt_id": "c1ec0f53-463e-4024-8235-7274113487f6", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00706, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.95698898693081, "model_name": "gemini-3.1-pro-preview", "output_tokens": 399, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:55.696618+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00706, "created_at": "2026-09-17T03:26:55.696Z" }, { "sequence": 3393, "attempt_id": "9959c831-7a83-44af-a761-6bcb7ab9b4c4", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003366, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.6129144199658185, "model_name": "gemini-3.1-pro-preview", "output_tokens": 91, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:56.341688+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003366, "created_at": "2026-09-17T03:26:56.341Z" }, { "sequence": 3394, "attempt_id": "5925795a-f739-4591-aea2-87a9c3bcfe8c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 0.7782481519971043, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:56.656374+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004944, "created_at": "2026-09-17T03:26:56.656Z" }, { "sequence": 3395, "attempt_id": "1044cfeb-8171-4dfd-afe4-69638024f375", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001239, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.6576161490520462, "model_name": "gemini-3.8-flash", "output_tokens": 103, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:56.839701+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001239, "created_at": "2026-09-17T03:26:56.841Z" }, { "sequence": 3396, "attempt_id": "4b712298-d961-4478-8438-df43d87426b6", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.055056803044863, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:57.014901+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02458, "created_at": "2026-09-17T03:26:57.015Z" }, { "sequence": 3397, "attempt_id": "9d2b4fe2-48d5-4cae-857f-0c3c902c3acc", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00486375, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.945031356997788, "model_name": "muse-spark-1.2", "output_tokens": 400, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:57.569708+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00486375, "created_at": "2026-09-17T03:26:57.569Z" }, { "sequence": 3398, "attempt_id": "a678f07b-775a-4bc1-a7d5-72e30024cd95", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.2464488060213625, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:58.452162+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003555, "created_at": "2026-09-17T03:26:58.452Z" }, { "sequence": 3399, "attempt_id": "e2fc7816-0072-456d-bdc7-c930014f284e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00466825, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 4.481364218052477, "model_name": "muse-spark-1.3", "output_tokens": 354, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:58.758362+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00466825, "created_at": "2026-09-17T03:26:58.758Z" }, { "sequence": 3400, "attempt_id": "c1011e2b-a2a9-42d2-8d05-cfa62ce93352", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0032175, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 2.956240388099104, "model_name": "gemini-3.8-flash", "output_tokens": 631, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:59.796296+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0032175, "created_at": "2026-09-17T03:26:59.796Z" }, { "sequence": 3401, "attempt_id": "ca5c4fa9-612e-4ac8-8f47-d57293dfabad", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007188, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 4.563923442969099, "model_name": "gpt-5.6-terra", "output_tokens": 201, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:59.991382+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007188, "created_at": "2026-09-17T03:26:59.991Z" }, { "sequence": 3402, "attempt_id": "dec815b6-86f5-4f83-92b1-de3c26addc8d", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00728775, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.588059064000845, "model_name": "muse-spark-1.3", "output_tokens": 948, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:27:00.181396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00728775, "created_at": "2026-09-17T03:27:00.181Z" }, { "sequence": 3403, "attempt_id": "333f2ffa-b3ef-4395-816b-2f21fa593f6b", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.5934550740057603, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:00.361705+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:27:00.362Z" }, { "sequence": 3404, "attempt_id": "51f5392a-543e-426c-bd1b-f27b7890e395", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.2841074910247698, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:00.546678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:27:00.546Z" }, { "sequence": 3405, "attempt_id": "7bfaf621-c735-4c7f-9fcc-2a9440f6961d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 4.478169625042938, "model_name": "gemini-3.1-pro-preview", "output_tokens": 291, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:01.016475+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T03:27:01.016Z" }, { "sequence": 3406, "attempt_id": "a9627db5-10c5-4da7-8fea-dc42e280019c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.0297149729449302, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:01.197688+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004928, "created_at": "2026-09-17T03:27:01.197Z" }, { "sequence": 3407, "attempt_id": "35e2c833-ef0c-40dd-8cea-41d4f3153b22", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.2063706510234624, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:01.378464+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:27:01.379Z" }, { "sequence": 3408, "attempt_id": "e2bd199c-d190-41dc-ad50-ec6000b4888f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8544028759934008, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:01.585599+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:27:01.585Z" }, { "sequence": 3409, "attempt_id": "cabd05a6-95b3-4668-af31-28b28bb0e097", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0065965, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 10.055344258085825, "model_name": "muse-spark-1.3", "output_tokens": 808, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:01.843812+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0065965, "created_at": "2026-09-17T03:27:01.843Z" }, { "sequence": 3410, "attempt_id": "f200f730-3c13-4894-867c-c6e878674218", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0015165, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.2596789749804884, "model_name": "gemini-3.8-flash", "output_tokens": 179, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:02.024802+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0015165, "created_at": "2026-09-17T03:27:02.025Z" }, { "sequence": 3411, "attempt_id": "5e8642d5-0f06-44c2-b18a-9e7329f17e8c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015592, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 7.569807694060728, "model_name": "gpt-5.6-sol", "output_tokens": 302, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:02.756976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015592, "created_at": "2026-09-17T03:27:02.757Z" }, { "sequence": 3412, "attempt_id": "c821fd24-152b-4bda-9565-fbf262f3dc36", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038155, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.5707673049764708, "model_name": "muse-spark-1.2", "output_tokens": 156, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:02.949507+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0038155, "created_at": "2026-09-17T03:27:02.949Z" }, { "sequence": 3413, "attempt_id": "e69698c0-9cf7-487f-9ec5-5cd12e6696e2", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 0.8687157579697669, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:03.129047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004954, "created_at": "2026-09-17T03:27:03.129Z" }, { "sequence": 3414, "attempt_id": "91157dfe-ead9-4309-81d2-b3a27b6766b1", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.7542114569805562, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:03.327282+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02463, "created_at": "2026-09-17T03:27:03.327Z" }, { "sequence": 3415, "attempt_id": "407c5441-d160-4ff6-ac99-e9a7d3a236d6", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041725, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 2.606167222955264, "model_name": "muse-spark-1.3", "output_tokens": 240, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:03.804197+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0041725, "created_at": "2026-09-17T03:27:03.804Z" }, { "sequence": 3416, "attempt_id": "ef6b399b-fee4-4099-b52c-2066442b741c", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-gridspan-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.8249003910459578, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:04.147322+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003695, "created_at": "2026-09-17T03:27:04.147Z" }, { "sequence": 3417, "attempt_id": "ac6c7dda-6d78-433d-b43f-13c5d05459d8", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00139125, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.316587240085937, "model_name": "gemini-3.8-flash", "output_tokens": 143, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:27:04.445931+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00139125, "created_at": "2026-09-17T03:27:04.446Z" }, { "sequence": 3418, "attempt_id": "27be6c5c-b17e-4f9c-b5e2-ac14242a5f4d", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006586, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.543350588995963, "model_name": "gemini-3.1-pro-preview", "output_tokens": 361, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:04.905402+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006586, "created_at": "2026-09-17T03:27:04.905Z" }, { "sequence": 3419, "attempt_id": "c77d3965-4898-46f7-a99a-06f551d7d005", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005494, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 3.166271475958638, "model_name": "gpt-5.6-terra", "output_tokens": 59, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:05.191368+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005494, "created_at": "2026-09-17T03:27:05.191Z" }, { "sequence": 3420, "attempt_id": "4c2b3fab-37b5-47dd-99d5-4b152534672a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.9641968479845673, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:05.601588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00988, "created_at": "2026-09-17T03:27:05.601Z" }, { "sequence": 3421, "attempt_id": "ae94a838-7528-4068-a5f5-1eef42cd6d41", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 4.316111703985371, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:06.079534+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009852, "created_at": "2026-09-17T03:27:06.079Z" }, { "sequence": 3422, "attempt_id": "cafe4e99-1d6e-4c9d-acba-b7b09ad8c056", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.8264181669801474, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:06.263073+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0247, "created_at": "2026-09-17T03:27:06.263Z" }, { "sequence": 3423, "attempt_id": "2063b8e1-b727-4f09-bd20-c9d356f34211", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005544, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.116866817930713, "model_name": "gemini-3.1-pro-preview", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:07.054398+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005544, "created_at": "2026-09-17T03:27:07.054Z" }, { "sequence": 3424, "attempt_id": "8660b526-3747-42df-a08c-de20ae845de1", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7693761099362746, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:07.238917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003725, "created_at": "2026-09-17T03:27:07.240Z" }, { "sequence": 3425, "attempt_id": "42073d74-9ab0-43fc-9d15-070364029b64", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005664, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.563726330990903, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:07.650163+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005664, "created_at": "2026-09-17T03:27:07.650Z" }, { "sequence": 3426, "attempt_id": "d1c5384d-0f24-4e4f-84ec-73a4c2138be9", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.425033177016303, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:07.836633+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004968, "created_at": "2026-09-17T03:27:07.836Z" }, { "sequence": 3427, "attempt_id": "31b63c80-53cd-4564-8f07-aa749af7c7e8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004856, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.192412376985885, "model_name": "muse-spark-1.3", "output_tokens": 397, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:08.011045+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004856, "created_at": "2026-09-17T03:27:08.011Z" }, { "sequence": 3428, "attempt_id": "64a9a54e-0da1-4827-bc54-59b4d60bcc81", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006029, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.525265312055126, "model_name": "muse-spark-1.2", "output_tokens": 673, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:08.510401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006029, "created_at": "2026-09-17T03:27:08.510Z" }, { "sequence": 3429, "attempt_id": "48b7ccb3-183e-4770-a3da-c058e2d6b5e1", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0021, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.5289983499096707, "model_name": "gemini-3.8-flash", "output_tokens": 330, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:08.792396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0021, "created_at": "2026-09-17T03:27:08.792Z" }, { "sequence": 3430, "attempt_id": "3381b685-fb64-461c-bf9e-2f53241d1cea", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.840896283974871, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:09.048603+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:27:09.048Z" }, { "sequence": 3431, "attempt_id": "63c38b7f-8777-4b45-b3cf-7d960cd3d397", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.8240471970057115, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:09.517894+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:27:09.518Z" }, { "sequence": 3432, "attempt_id": "7fa9b3ee-a211-4b33-9746-2b5fc1216e9a", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.5039393350016326, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:09.705856+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:27:09.705Z" }, { "sequence": 3433, "attempt_id": "f1fdf8f8-88f4-4dcb-ac2e-9a48dbb5e68d", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9373363140039146, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:09.885214+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:27:09.885Z" }, { "sequence": 3434, "attempt_id": "466995f8-28d1-4a31-8ab0-d94d40a40a02", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01128725, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 12.052449577953666, "model_name": "muse-spark-1.2", "output_tokens": 1912, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:10.059027+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01128725, "created_at": "2026-09-17T03:27:10.059Z" }, { "sequence": 3435, "attempt_id": "b4d7e2f7-23b6-46ba-9de3-46c3cf603e43", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00704, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.450001001008786, "model_name": "gemini-3.1-pro-preview", "output_tokens": 395, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:10.258695+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00704, "created_at": "2026-09-17T03:27:10.260Z" }, { "sequence": 3436, "attempt_id": "6abb5e43-75cb-4d07-b136-63ceaf1e1fc8", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.788845579023473, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:10.489330+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:27:10.489Z" }, { "sequence": 3437, "attempt_id": "e67ce418-3d21-44d9-88eb-72ab632eb327", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.249412951990962, "model_name": "gemini-3.8-flash", "output_tokens": 99, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:10.669277+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00122175, "created_at": "2026-09-17T03:27:10.669Z" }, { "sequence": 3438, "attempt_id": "83638b6d-cc4f-49aa-aaac-fb33d4dd1262", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003204, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.273903712979518, "model_name": "gemini-3.1-pro-preview", "output_tokens": 78, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:11.249072+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003204, "created_at": "2026-09-17T03:27:11.249Z" }, { "sequence": 3439, "attempt_id": "6dc68cf5-f16f-4025-8c6a-5feb1dd0d8d2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041205, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.2397822729544714, "model_name": "muse-spark-1.2", "output_tokens": 226, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:11.492749+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0041205, "created_at": "2026-09-17T03:27:11.492Z" }, { "sequence": 3440, "attempt_id": "ce02f5bf-cb7b-4e9b-a605-0d1d64485431", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2041458709863946, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:11.684944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00984, "created_at": "2026-09-17T03:27:11.685Z" }, { "sequence": 3441, "attempt_id": "4db30288-efae-41ce-b0e7-b1d19f73b32f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0593727179802954, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:11.867472+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004948, "created_at": "2026-09-17T03:27:11.867Z" }, { "sequence": 3442, "attempt_id": "cd8f33fe-ca95-45e5-bcb6-490dcbd5c005", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9956335800234228, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:12.055869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004948, "created_at": "2026-09-17T03:27:12.056Z" }, { "sequence": 3443, "attempt_id": "b0fd7471-8b1b-4f8f-8dfb-584619c57e80", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.693875737953931, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:12.240287+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0246, "created_at": "2026-09-17T03:27:12.240Z" }, { "sequence": 3444, "attempt_id": "e25ffd6f-4f4d-4aeb-9a85-6ddf3f15d7f4", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070565, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.397191503085196, "model_name": "muse-spark-1.2", "output_tokens": 913, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:12.828268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0070565, "created_at": "2026-09-17T03:27:12.828Z" }, { "sequence": 3445, "attempt_id": "79c42f2b-d471-4118-af95-ca7049a6bdd5", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001359, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.237039059982635, "model_name": "gemini-3.8-flash", "output_tokens": 135, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:13.025397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001359, "created_at": "2026-09-17T03:27:13.025Z" }, { "sequence": 3446, "attempt_id": "e089df77-ec50-4aac-bc50-e104fa51c82d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.2238846860127524, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:13.209375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003561, "created_at": "2026-09-17T03:27:13.211Z" }, { "sequence": 3447, "attempt_id": "4a44cf74-9eee-41ac-9414-df2240b5372a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004639, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.835718263988383, "model_name": "muse-spark-1.3", "output_tokens": 348, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:13.721309+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004639, "created_at": "2026-09-17T03:27:13.721Z" }, { "sequence": 3448, "attempt_id": "20ff534c-72dd-422c-a1e2-838db08ef2de", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.225820800056681, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:14.251527+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:27:14.251Z" }, { "sequence": 3449, "attempt_id": "9d9c9d5d-fe90-4f69-989a-a45ea25ed2c1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00482975, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.290991888032295, "model_name": "muse-spark-1.2", "output_tokens": 392, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:14.531564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00482975, "created_at": "2026-09-17T03:27:14.531Z" }, { "sequence": 3450, "attempt_id": "b11d40a0-8f59-40e0-b493-f4af0882e42f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004146, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.2766582779586315, "model_name": "gemini-3.1-pro-preview", "output_tokens": 156, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:14.718527+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004146, "created_at": "2026-09-17T03:27:14.718Z" }, { "sequence": 3451, "attempt_id": "792c458e-cb57-4530-adeb-d6a0d150a695", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008937, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 17.539160526008345, "model_name": "muse-spark-1.3", "output_tokens": 1359, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:14.906361+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008937, "created_at": "2026-09-17T03:27:14.906Z" }, { "sequence": 3452, "attempt_id": "41ff4d85-bc12-4943-b6b7-31f1c72a0708", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006012, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.077666606986895, "model_name": "gpt-5.6-terra", "output_tokens": 98, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:15.287349+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006012, "created_at": "2026-09-17T03:27:15.287Z" }, { "sequence": 3453, "attempt_id": "f5bcf644-124b-4597-b5d6-06deed02b389", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.8020970849320292, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:15.520901+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003764, "created_at": "2026-09-17T03:27:15.521Z" }, { "sequence": 3454, "attempt_id": "d966a7ac-af89-469a-bfd9-4cb163f9eb42", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02718, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.1983023890061304, "model_name": "gpt-6-astra", "output_tokens": 60, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:15.703504+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02718, "created_at": "2026-09-17T03:27:15.703Z" }, { "sequence": 3455, "attempt_id": "0cad11f1-c03d-424d-b17e-486b554ce555", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00482975, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.74847200489603, "model_name": "muse-spark-1.3", "output_tokens": 392, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:15.983095+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00482975, "created_at": "2026-09-17T03:27:15.983Z" }, { "sequence": 3456, "attempt_id": "1129de2d-7e4c-4ad9-9fce-00ca5f4e343c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006228, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.332309229997918, "model_name": "gpt-5.6-luna", "output_tokens": 116, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:16.725261+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0006228, "created_at": "2026-09-17T03:27:16.725Z" }, { "sequence": 3457, "attempt_id": "624b604b-5370-4c9c-b251-a66cb74f1f56", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9126166520873085, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:16.910874+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004942, "created_at": "2026-09-17T03:27:16.911Z" }, { "sequence": 3458, "attempt_id": "dcfb7a49-9ba7-4d22-b784-6603e955ec79", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.3804174499819055, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:17.544269+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004942000000000001, "created_at": "2026-09-17T03:27:17.544Z" }, { "sequence": 3459, "attempt_id": "4cca6659-41e2-4ca0-b3f2-20b71164767d", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002556, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 3.1547752380138263, "model_name": "gemini-3.8-flash", "output_tokens": 449, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:27:17.742631+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.002556, "created_at": "2026-09-17T03:27:17.742Z" }, { "sequence": 3460, "attempt_id": "9f68be40-8679-424c-9251-d93cc46a6309", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.403071120963432, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:18.106869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009828, "created_at": "2026-09-17T03:27:18.107Z" }, { "sequence": 3461, "attempt_id": "4041a764-742d-4e15-ad95-9828f03fe4ea", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.3503176809754223, "model_name": "gemini-3.8-flash", "output_tokens": 99, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:18.442753+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00122175, "created_at": "2026-09-17T03:27:18.442Z" }, { "sequence": 3462, "attempt_id": "09971ea1-be0e-4bf1-bd64-309a3faa645f", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 3.040577891981229, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:18.641372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02457, "created_at": "2026-09-17T03:27:18.642Z" }, { "sequence": 3463, "attempt_id": "f94eca5d-d1a3-4d2b-8bf8-cf0950fb1642", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.781163937994279, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:18.820648+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003677, "created_at": "2026-09-17T03:27:18.820Z" }, { "sequence": 3464, "attempt_id": "3e94ab9c-14a3-4dbe-9fec-d10dedd9e7e7", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006922, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.886587774963118, "model_name": "gemini-3.1-pro-preview", "output_tokens": 383, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:27:19.009422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006922, "created_at": "2026-09-17T03:27:19.009Z" }, { "sequence": 3465, "attempt_id": "a183abb2-6ae2-4f3f-b6fb-c1372ed25eb1", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004163, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.057666338980198, "model_name": "muse-spark-1.3", "output_tokens": 236, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:19.983022+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004163, "created_at": "2026-09-17T03:27:19.983Z" }, { "sequence": 3466, "attempt_id": "947aad97-56d7-4626-bb95-c0172d21cf4f", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003288, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.317415529047139, "model_name": "gemini-3.1-pro-preview", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:20.228602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.003288, "created_at": "2026-09-17T03:27:20.228Z" }, { "sequence": 3467, "attempt_id": "a525ad88-9e9e-4feb-b6a4-60d3681522c3", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.7456459649838507, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:20.417281+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0245, "created_at": "2026-09-17T03:27:20.417Z" }, { "sequence": 3468, "attempt_id": "f8974b1c-5559-4376-92dc-47fedb16957b", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00436275, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.0873870269861072, "model_name": "muse-spark-1.2", "output_tokens": 283, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:20.592352+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00436275, "created_at": "2026-09-17T03:27:20.592Z" }, { "sequence": 3469, "attempt_id": "28237990-84dc-4956-86cc-c97412dc1cde", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005276, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.7679162779822946, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:20.793035+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005276, "created_at": "2026-09-17T03:27:20.793Z" }, { "sequence": 3470, "attempt_id": "46a6a20d-1843-4837-a4e6-8d29be27ce57", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.6643419649917632, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:20.969127+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0098, "created_at": "2026-09-17T03:27:20.969Z" }, { "sequence": 3471, "attempt_id": "174a0dd6-6bfa-471a-9537-776819f53f67", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.952761990018189, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:21.158653+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004928, "created_at": "2026-09-17T03:27:21.158Z" }, { "sequence": 3472, "attempt_id": "816e3625-6c9d-4ba6-8e89-f3eb676b9db2", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0078725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 13.822517538908869, "model_name": "muse-spark-1.3", "output_tokens": 1105, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:21.345887+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0078725, "created_at": "2026-09-17T03:27:21.346Z" }, { "sequence": 3473, "attempt_id": "bb247ae8-b660-413a-9512-541e3d7a3a5d", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.0269477660767734, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:21.814142+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00036559999999999994, "created_at": "2026-09-17T03:27:21.814Z" }, { "sequence": 3474, "attempt_id": "d021020f-f312-453f-aaca-8350c011a18f", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0014865, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.4603587160818279, "model_name": "gemini-3.8-flash", "output_tokens": 171, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:21.998549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0014865, "created_at": "2026-09-17T03:27:21.998Z" }, { "sequence": 3475, "attempt_id": "bc687b50-b19c-4e92-a16f-63d1768e8116", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0037985, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.5868755269329995, "model_name": "muse-spark-1.3", "output_tokens": 152, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:22.380221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0037985, "created_at": "2026-09-17T03:27:22.380Z" }, { "sequence": 3476, "attempt_id": "3aedf5d7-7370-4d87-914e-cc68b824e7ad", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005624, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.3746887859888375, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:22.905902+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005624, "created_at": "2026-09-17T03:27:22.906Z" }, { "sequence": 3477, "attempt_id": "2109e751-4e1e-47e3-8e7c-a53d27623fd2", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.9234145240625367, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:23.092360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02474, "created_at": "2026-09-17T03:27:23.092Z" }, { "sequence": 3478, "attempt_id": "35833890-c46c-4a3e-868c-3b1466d6acf8", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00412575, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 2.2771029549185187, "model_name": "muse-spark-1.2", "output_tokens": 229, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:23.429258+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00412575, "created_at": "2026-09-17T03:27:23.429Z" }, { "sequence": 3479, "attempt_id": "520223fd-4ed9-43aa-b0a7-51be296d7aa8", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004834, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 3.449896131060086, "model_name": "gemini-3.1-pro-preview", "output_tokens": 215, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:23.627600+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004834, "created_at": "2026-09-17T03:27:23.629Z" }, { "sequence": 3480, "attempt_id": "f6c2a3a7-5fe8-43bb-81df-6fd5c3598ad7", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003615, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.7910261900397018, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:23.883685+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003615, "created_at": "2026-09-17T03:27:23.883Z" }, { "sequence": 3481, "attempt_id": "f17f62af-b950-427d-b70f-b02bdd24332e", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012264, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 9.5531799410237, "model_name": "muse-spark-1.2", "output_tokens": 2133, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:27:24.653611+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.012264, "created_at": "2026-09-17T03:27:24.653Z" }, { "sequence": 3482, "attempt_id": "c1bc120c-c83c-43e3-9bd0-5b5691536daa", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.761120575014502, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:24.852566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004976000000000001, "created_at": "2026-09-17T03:27:24.852Z" }, { "sequence": 3483, "attempt_id": "bab85f39-daa9-4958-a847-88458da0a421", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00217125, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 2.4632702890085056, "model_name": "gemini-3.8-flash", "output_tokens": 348, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:25.037485+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00217125, "created_at": "2026-09-17T03:27:25.038Z" }, { "sequence": 3484, "attempt_id": "51b42f9b-4b4d-432e-a673-53e9536952c8", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.8008321789093316, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:25.612189+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02533, "created_at": "2026-09-17T03:27:25.612Z" }, { "sequence": 3485, "attempt_id": "7e6c6a42-5a6b-49e5-8937-f046c14b2bb8", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005358, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.185327217914164, "model_name": "gpt-5.6-terra", "output_tokens": 36, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:26.038234+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005358, "created_at": "2026-09-17T03:27:26.038Z" }, { "sequence": 3486, "attempt_id": "e872bf2f-9c43-4e3e-8276-abd8df89d6c6", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.032517412910238, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:26.240270+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010132, "created_at": "2026-09-17T03:27:26.240Z" }, { "sequence": 3487, "attempt_id": "9969a4ca-f34b-47e1-b84e-8e8223488027", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004401, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 3.037510168971494, "model_name": "muse-spark-1.3", "output_tokens": 287, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:26.426489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.004401, "created_at": "2026-09-17T03:27:26.426Z" }, { "sequence": 3488, "attempt_id": "cc5c74cc-b727-45bc-ba67-6f07cb35a9bb", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00556125, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 2.897013955982402, "model_name": "muse-spark-1.2", "output_tokens": 560, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:26.608652+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00556125, "created_at": "2026-09-17T03:27:26.610Z" }, { "sequence": 3489, "attempt_id": "8642b4fc-debc-47fa-9cbd-acb6dc898ddf", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005094, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.867822211002931, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:26.905609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005094, "created_at": "2026-09-17T03:27:26.905Z" }, { "sequence": 3490, "attempt_id": "4b9b478c-d207-460d-add9-41d3462cf678", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005754, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 4.808205430046655, "model_name": "gemini-3.1-pro-preview", "output_tokens": 287, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:27.090812+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005754, "created_at": "2026-09-17T03:27:27.091Z" }, { "sequence": 3491, "attempt_id": "1718f6e8-8112-4e13-8acc-5783d0e9c570", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.7582473671063781, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:27.267841+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003905, "created_at": "2026-09-17T03:27:27.267Z" }, { "sequence": 3492, "attempt_id": "f03d62d0-993d-48e7-8066-523565701ea0", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00169875, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 2.058656213921495, "model_name": "gemini-3.8-flash", "output_tokens": 211, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:27.861819+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00169875, "created_at": "2026-09-17T03:27:27.862Z" }, { "sequence": 3493, "attempt_id": "a63c2871-bcde-4bdc-9dee-44cba40f459f", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-textalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.0645588070619851, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:28.534819+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0005004, "created_at": "2026-09-17T03:27:28.535Z" }, { "sequence": 3494, "attempt_id": "4fe30b32-a450-4207-b2f9-cf04eb15f1a3", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-textalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.0618057959945872, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:28.866608+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02488, "created_at": "2026-09-17T03:27:28.866Z" }, { "sequence": 3495, "attempt_id": "51afe80f-986d-4d6b-a00c-c364b8b59ade", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-textalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005364, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4801565950037912, "model_name": "gpt-5.6-terra", "output_tokens": 44, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:29.055055+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005364, "created_at": "2026-09-17T03:27:29.055Z" }, { "sequence": 3496, "attempt_id": "7c3512b4-5968-4d9d-a3ea-a270461f3a1c", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010996, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 7.491445573978126, "model_name": "gpt-5.6-sol", "output_tokens": 69, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:29.245079+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010996, "created_at": "2026-09-17T03:27:29.245Z" }, { "sequence": 3497, "attempt_id": "19115295-5ba3-4345-bc28-b2cd8da7359f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051475, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.633492987952195, "model_name": "muse-spark-1.2", "output_tokens": 445, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:29.433098+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0051475, "created_at": "2026-09-17T03:27:29.434Z" }, { "sequence": 3498, "attempt_id": "b499a767-1e34-434e-b5dd-b6351f239df3", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005048, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.8776702269678935, "model_name": "gemini-3.1-pro-preview", "output_tokens": 219, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:29.611534+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005048, "created_at": "2026-09-17T03:27:29.611Z" }, { "sequence": 3499, "attempt_id": "31e59e81-e364-48ea-8ff7-8515bce11e42", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-textalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.9951251980382949, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:30.050478+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003764, "created_at": "2026-09-17T03:27:30.050Z" }, { "sequence": 3500, "attempt_id": "eaf4e7ec-c780-4f11-8218-29fee784de7b", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-textalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.5989189210813493, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:30.690098+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009952, "created_at": "2026-09-17T03:27:30.690Z" }, { "sequence": 3501, "attempt_id": "3d8c5ca0-8672-4b68-9f42-d2aba14d226c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00515175, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.495390837895684, "model_name": "muse-spark-1.3", "output_tokens": 446, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:30.922179+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00515175, "created_at": "2026-09-17T03:27:30.922Z" }, { "sequence": 3502, "attempt_id": "01d3a111-9a64-4ba3-83ef-80cbe8ad0bf5", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-terra", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005692, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.5033974880352616, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:31.752513+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005692, "created_at": "2026-09-17T03:27:31.752Z" }, { "sequence": 3503, "attempt_id": "7884196c-c265-4c42-9622-768d52fe5d13", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-luna", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004996, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.174609418027103, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:32.050094+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0004996, "created_at": "2026-09-17T03:27:32.050Z" }, { "sequence": 3504, "attempt_id": "f867c7c4-381b-46c0-b0d7-c27d7004c55e", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-textalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005586, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.200185633962974, "model_name": "gemini-3.8-flash", "output_tokens": 1257, "prediction": { "choice": "A" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:32.928536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005586, "created_at": "2026-09-17T03:27:32.928Z" }, { "sequence": 3505, "attempt_id": "321241db-96d1-4378-b99d-c7616561be2f", "run_id": "qualitative-20260917", "model_id": "gemini-3.5-flash-lite", "task_id": "layoutbench-padcompare-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.8968357000267133, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:33.134226+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0003761, "created_at": "2026-09-17T03:27:33.134Z" }, { "sequence": 3506, "attempt_id": "81567396-d60f-4b12-856d-d48e04ea1246", "run_id": "qualitative-20260917", "model_id": "gemini-3.8-flash", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00253275, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 2.0958253300050274, "model_name": "gemini-3.8-flash", "output_tokens": 443, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:34.034140+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00253275, "created_at": "2026-09-17T03:27:34.034Z" }, { "sequence": 3507, "attempt_id": "7281338a-a1ec-4d0e-97b6-5fea06cea4fb", "run_id": "qualitative-20260917", "model_id": "gpt-5.6-sol", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011176, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 4.56689106207341, "model_name": "gpt-5.6-sol", "output_tokens": 76, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:34.380198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011176, "created_at": "2026-09-17T03:27:34.380Z" }, { "sequence": 3508, "attempt_id": "7331b161-147e-41f7-93ea-10362db8e7af", "run_id": "qualitative-20260917", "model_id": "gpt-6-astra", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 5.212774676969275, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:35.019911+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02484, "created_at": "2026-09-17T03:27:35.020Z" }, { "sequence": 3509, "attempt_id": "3de77d8e-2379-4cc0-8c15-d58bcf7b4d93", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-textalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0088215, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.541983081027865, "model_name": "muse-spark-1.2", "output_tokens": 1323, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:36.975407+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0088215, "created_at": "2026-09-17T03:27:36.975Z" }, { "sequence": 3510, "attempt_id": "019c1509-c422-4e64-b5c6-471eeee033ac", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-textalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.009106, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 10.01918929698877, "model_name": "gemini-3.1-pro-preview", "output_tokens": 565, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:27:38.065625+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009106, "created_at": "2026-09-17T03:27:38.065Z" }, { "sequence": 3511, "attempt_id": "a4735712-7188-449b-9e11-1f0fd5bdf3e7", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.2", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00657375, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 4.964360686019063, "model_name": "muse-spark-1.2", "output_tokens": 795, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:38.280195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00657375, "created_at": "2026-09-17T03:27:38.280Z" }, { "sequence": 3512, "attempt_id": "ec2249ec-37b5-49f3-9ccf-f79a577d1281", "run_id": "qualitative-20260917", "model_id": "gemini-3.1-pro-preview", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01706, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 11.076936650904827, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1228, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:42.197580+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01706, "created_at": "2026-09-17T03:27:42.197Z" }, { "sequence": 3513, "attempt_id": "002c4e05-3bd5-43f3-8d8b-1b2399e1066f", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00507775, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 10.016927120042965, "model_name": "muse-spark-1.3", "output_tokens": 443, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:43.145125+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00507775, "created_at": "2026-09-17T03:27:43.145Z" }, { "sequence": 3514, "attempt_id": "d280c4d2-a69f-4b64-9b0d-e49cf033ea0a", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0135305, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 21.903505982016213, "model_name": "muse-spark-1.3", "output_tokens": 2431, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:27:51.330736+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0135305, "created_at": "2026-09-17T03:27:51.330Z" }, { "sequence": 3515, "attempt_id": "4c8f650c-06b6-40e7-a0b1-537f34ad433c", "run_id": "qualitative-20260917", "model_id": "muse-spark-1.3", "task_id": "layoutbench-textalign-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02359025, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 48.129503038013354, "model_name": "muse-spark-1.3", "output_tokens": 4798, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:28:03.036188+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02359025, "created_at": "2026-09-17T03:28:03.036Z" }, { "sequence": 3516, "attempt_id": "ddf604be-9e7c-40da-98d8-432eb151e3c1", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.8769601780222729, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:35.499249+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T12:08:35.499Z" }, { "sequence": 3517, "attempt_id": "a0225c89-a638-40c0-a599-8a4063aabc15", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 1.1290975629817694, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:35.724880+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001876, "created_at": "2026-09-17T12:08:35.725Z" }, { "sequence": 3518, "attempt_id": "308a6759-f4ce-4f06-9a67-93d5dc59e33f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.14046645804774, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:36.727083+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005798, "created_at": "2026-09-17T12:08:36.727Z" }, { "sequence": 3519, "attempt_id": "850d8254-f8dc-4a64-b360-f081fbb7eb82", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.182488483027555, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:36.912573+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00571, "created_at": "2026-09-17T12:08:36.912Z" }, { "sequence": 3520, "attempt_id": "cded12e8-28a2-4246-b4ad-727a6326b1a2", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.9681625210214406, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:37.088210+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:08:37.088Z" }, { "sequence": 3521, "attempt_id": "25fa8bb7-43a3-472e-b150-e6bed6ee1116", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.3426973309833556, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:37.292010+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T12:08:37.293Z" }, { "sequence": 3522, "attempt_id": "ca4661fe-80b0-4085-82a6-8c7685def6f7", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.3124748109839857, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:37.466205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014275, "created_at": "2026-09-17T12:08:37.466Z" }, { "sequence": 3523, "attempt_id": "e62cf3d1-3c0b-469f-bde8-637477a36e50", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7125317560276017, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:37.652188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T12:08:37.652Z" }, { "sequence": 3524, "attempt_id": "16586658-3f52-4069-ae2b-ef7c20146db5", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.1109944520285353, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:37.846785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T12:08:37.848Z" }, { "sequence": 3525, "attempt_id": "0baa5a25-5abd-4c80-a360-31eff9517028", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01562, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.7517399759963155, "model_name": "claude-opus-5", "output_tokens": 56, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:38.331114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01562, "created_at": "2026-09-17T12:08:38.331Z" }, { "sequence": 3526, "attempt_id": "465e46d2-9388-48d4-a697-ed3dbeeeb72a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 3.815051767975092, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:38.517985+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02901, "created_at": "2026-09-17T12:08:38.518Z" }, { "sequence": 3527, "attempt_id": "ce780fd7-3836-4eca-8922-ae1d2711ca92", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.1542429350083694, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:38.799934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T12:08:38.800Z" }, { "sequence": 3528, "attempt_id": "ec8b158f-34ac-4d8d-b3e6-289b1fc616db", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.7212790919002146, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:39.239651+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001876, "created_at": "2026-09-17T12:08:39.239Z" }, { "sequence": 3529, "attempt_id": "4a05dbaf-c092-4d99-a314-b5b3b2170f70", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.0464949948946014, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:39.425397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T12:08:39.425Z" }, { "sequence": 3530, "attempt_id": "52cca9b9-8b32-4d5d-8849-5959e7929658", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 5.405033276998438, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:40.005945+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02857, "created_at": "2026-09-17T12:08:40.006Z" }, { "sequence": 3531, "attempt_id": "5f0e3833-8609-465d-b2da-e3d22d715f5c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 3.2748661399818957, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:40.560434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T12:08:40.560Z" }, { "sequence": 3532, "attempt_id": "8ce5aadd-2d8e-4a03-8ae4-42c93040f464", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01552, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.9586243729572743, "model_name": "claude-opus-5", "output_tokens": 52, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:40.805845+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01552, "created_at": "2026-09-17T12:08:40.806Z" }, { "sequence": 3533, "attempt_id": "0856042b-eb44-48c9-9866-06743dd39fc0", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.9459432050352916, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:41.013191+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T12:08:41.013Z" }, { "sequence": 3534, "attempt_id": "9c5b3ae2-ec4a-48b9-a90b-b2d6c10cf45e", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.2964780090842396, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:41.202065+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T12:08:41.202Z" }, { "sequence": 3535, "attempt_id": "b41422bc-f6af-4b89-ba16-2b43df407d45", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006848, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.273903591092676, "model_name": "claude-sonnet-5", "output_tokens": 116, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:41.391503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006848, "created_at": "2026-09-17T12:08:41.391Z" }, { "sequence": 3536, "attempt_id": "e2f526e2-b811-43da-b594-9c9056e09f83", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.3149605629732832, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:41.565399+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T12:08:41.565Z" }, { "sequence": 3537, "attempt_id": "3864e92a-d07d-4c56-b6e5-aae8da9bc4ab", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9978810040047392, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:41.764810+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T12:08:41.765Z" }, { "sequence": 3538, "attempt_id": "1e85cb36-e61e-4f40-998e-21f52262c783", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 1.0323586100712419, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:42.417168+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T12:08:42.417Z" }, { "sequence": 3539, "attempt_id": "1930ae3f-ee91-4b00-b31a-4b5fb0038016", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.6535218720091507, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:42.667058+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00601, "created_at": "2026-09-17T12:08:42.667Z" }, { "sequence": 3540, "attempt_id": "79ab5cac-a9dc-4bd5-adbe-f8ae2296d9b8", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.222509662969969, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:42.937778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T12:08:42.937Z" }, { "sequence": 3541, "attempt_id": "c3b5c83d-b5ab-4ad0-a209-ed4d7f456bb7", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 1.160009917919524, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:43.117610+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T12:08:43.117Z" }, { "sequence": 3542, "attempt_id": "f07431e5-680d-4d40-b24d-6edaca5c1f92", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 5.780746399075724, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:43.433246+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02901, "created_at": "2026-09-17T12:08:43.433Z" }, { "sequence": 3543, "attempt_id": "f974b7c5-e6e9-4fcb-b337-2b228e29170d", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.7283834799891338, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:43.613607+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T12:08:43.613Z" }, { "sequence": 3544, "attempt_id": "87b655c9-bb7a-4ba6-9ae8-1a4bfc647fd9", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.9336639059474692, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:43.806869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014315, "created_at": "2026-09-17T12:08:43.807Z" }, { "sequence": 3545, "attempt_id": "9e72e0ed-ab6b-4cdd-90bf-277d3e5ae340", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6820524079957977, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:43.990551+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T12:08:43.990Z" }, { "sequence": 3546, "attempt_id": "b2ff857b-5285-417c-95be-da95c67af655", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.9703814279055223, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:44.173694+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T12:08:44.173Z" }, { "sequence": 3547, "attempt_id": "2115b9da-4950-4d2b-8dd3-76d9955ed0b1", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.7599883610382676, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:44.877942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T12:08:44.878Z" }, { "sequence": 3548, "attempt_id": "a0cefabc-48b3-46b6-aed8-24e26c564e8e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.068556701997295, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:45.060549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014315, "created_at": "2026-09-17T12:08:45.062Z" }, { "sequence": 3549, "attempt_id": "88543dbe-dd62-4ca1-957b-0c0ab2beffe3", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.7797020140569657, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:45.233319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T12:08:45.233Z" }, { "sequence": 3550, "attempt_id": "58f5bd4c-1f00-408a-97b6-44fb99be58d9", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.34217372094281, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:45.951262+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T12:08:45.951Z" }, { "sequence": 3551, "attempt_id": "108d01d3-63ec-4dae-a590-c1cbedad8640", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.183015643968247, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:46.138221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T12:08:46.139Z" }, { "sequence": 3552, "attempt_id": "65606005-6816-4083-b185-0c563037376f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 4.631953828036785, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:46.314032+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T12:08:46.314Z" }, { "sequence": 3553, "attempt_id": "4a32f575-bbb2-4d6c-96b0-d0a3989e7a64", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.0320713290711865, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:46.487840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T12:08:46.488Z" }, { "sequence": 3554, "attempt_id": "6472dccb-7ba7-4794-a8af-8dd656507c3d", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.1149664520053193, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:46.703944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T12:08:46.705Z" }, { "sequence": 3555, "attempt_id": "7bf44369-9d3a-421f-a283-1f218ac8f3c1", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 2.739163312013261, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:46.881361+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T12:08:46.881Z" }, { "sequence": 3556, "attempt_id": "cf37222f-ae65-4442-9407-2f9d62fc54e0", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8169512719614431, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:47.070156+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T12:08:47.071Z" }, { "sequence": 3557, "attempt_id": "715d51ad-ded9-42aa-86f4-d6b313d7e79d", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.7311281370930374, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:47.263913+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T12:08:47.264Z" }, { "sequence": 3558, "attempt_id": "b9d5c079-f75f-4cd4-99a2-1390359f69a3", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6499043039511889, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:47.444925+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T12:08:47.446Z" }, { "sequence": 3559, "attempt_id": "bc82d923-41f3-49e3-acde-7db34880f6bd", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.6370941780041903, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:48.005395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T12:08:48.005Z" }, { "sequence": 3560, "attempt_id": "a08f5a4c-f48b-4105-98ff-8499f6808d10", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6358918979531154, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:48.202141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T12:08:48.202Z" }, { "sequence": 3561, "attempt_id": "d32c5f72-1afa-4a60-8d41-ab9fa45f443f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.8325763709144667, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:48.530097+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T12:08:48.530Z" }, { "sequence": 3562, "attempt_id": "91ea5ecd-3f58-4c3f-821c-c34d484aefff", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.3158883000724018, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:49.006841+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T12:08:49.007Z" }, { "sequence": 3563, "attempt_id": "edb22c93-6aeb-461f-a371-21d6db197c16", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.6378387480508536, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:49.776422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T12:08:49.776Z" }, { "sequence": 3564, "attempt_id": "6d58ea2c-8f01-4c50-a725-204f59b00b79", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-007", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 1.287420937907882, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:49.999170+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T12:08:49.999Z" }, { "sequence": 3565, "attempt_id": "1f78f8a4-fe52-4773-93c6-0a692bad9f26", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017475, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.466560887056403, "model_name": "claude-opus-5", "output_tokens": 109, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:50.537157+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017475, "created_at": "2026-09-17T12:08:50.537Z" }, { "sequence": 3566, "attempt_id": "7bc8f142-b2a8-4813-ac2d-6918becd3413", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006306, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.7572364759398624, "model_name": "claude-sonnet-5", "output_tokens": 68, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:51.152841+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006306, "created_at": "2026-09-17T12:08:51.153Z" }, { "sequence": 3567, "attempt_id": "51435605-92bf-4d77-99a1-1e8419c9ad3e", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00778, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.901031270972453, "model_name": "claude-sonnet-5", "output_tokens": 188, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:51.354555+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00778, "created_at": "2026-09-17T12:08:51.354Z" }, { "sequence": 3568, "attempt_id": "b5bfe802-36a9-43d7-a101-0741bc2b98d2", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.167705429950729, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:51.535491+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T12:08:51.535Z" }, { "sequence": 3569, "attempt_id": "a0e4ac23-3440-4cd9-903f-9a9e31ca6549", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015615, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 3.0100152710219845, "model_name": "claude-opus-5", "output_tokens": 62, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:51.714143+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015615, "created_at": "2026-09-17T12:08:51.714Z" }, { "sequence": 3570, "attempt_id": "b9351743-e51e-4166-87b0-aac9e36fda25", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 4.257706481963396, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:51.927479+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T12:08:51.928Z" }, { "sequence": 3571, "attempt_id": "353f0d0e-9561-470a-bb02-8eea8fa70196", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.7783859580522403, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:52.104622+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001876, "created_at": "2026-09-17T12:08:52.104Z" }, { "sequence": 3572, "attempt_id": "0f6f62d7-14de-411f-8c70-0750eec3d0b6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.7056205479893833, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:52.633468+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:08:52.633Z" }, { "sequence": 3573, "attempt_id": "c0ab56ba-c402-4c01-b2f5-1095972e948d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.032725, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 12.566488765995018, "model_name": "claude-opus-5", "output_tokens": 719, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:53.308642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.032725, "created_at": "2026-09-17T12:08:53.308Z" }, { "sequence": 3574, "attempt_id": "73363b8e-4103-43ac-b7ec-6478588fbf07", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017845, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.519892710959539, "model_name": "claude-opus-5", "output_tokens": 145, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:53.506520+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017845, "created_at": "2026-09-17T12:08:53.507Z" }, { "sequence": 3575, "attempt_id": "4a1a7e5f-8735-4df7-8f9d-5895091283a3", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.761712172999978, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:53.693343+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T12:08:53.693Z" }, { "sequence": 3576, "attempt_id": "3cb95ce6-b649-4ce9-bc36-ec833f5ae3d0", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008358, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.786212519975379, "model_name": "claude-sonnet-5", "output_tokens": 267, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:53.969419+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008358, "created_at": "2026-09-17T12:08:53.969Z" }, { "sequence": 3577, "attempt_id": "672450ee-0287-4591-a717-2c81a13621d1", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.308228017995134, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:54.221983+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T12:08:54.222Z" }, { "sequence": 3578, "attempt_id": "bd145289-0ac6-47a3-b59b-8f3ff9081431", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.722560988040641, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "E" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T12:08:54.406660+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T12:08:54.406Z" }, { "sequence": 3579, "attempt_id": "1f9b0833-1b6d-4739-81d5-0ffceada2803", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01572, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.5095107610104606, "model_name": "claude-opus-5", "output_tokens": 42, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T12:08:54.807656+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01572, "created_at": "2026-09-17T12:08:54.807Z" }, { "sequence": 3580, "attempt_id": "dc6aec37-c5d1-4004-a3cd-b6df1123d5ee", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03931, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 5.922074805013835, "model_name": "claude-fable-5-1", "output_tokens": 217, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:55.115370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03931, "created_at": "2026-09-17T12:08:55.115Z" }, { "sequence": 3581, "attempt_id": "28836c1d-c602-48b4-ba1c-47fce038beda", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-002", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.7649832780007273, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:55.317079+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T12:08:55.317Z" }, { "sequence": 3582, "attempt_id": "4bf58c52-8953-4460-97e0-7726d861b855", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 4.405252791009843, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:55.760204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T12:08:55.760Z" }, { "sequence": 3583, "attempt_id": "457693a0-ef04-414f-aa69-5eb670c8fd56", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.706719771027565, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:08:56.024267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T12:08:56.026Z" }, { "sequence": 3584, "attempt_id": "85d3d8b8-640b-4b2d-bdd9-1bf5184afa66", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006598, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.9707984320120886, "model_name": "claude-sonnet-5", "output_tokens": 73, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:08:56.261983+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006598, "created_at": "2026-09-17T12:08:56.262Z" }, { "sequence": 3585, "attempt_id": "8d06ce44-1c8e-476c-bf40-a6c25e7cc912", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-002", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.6701229079626501, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:56.500454+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T12:08:56.501Z" }, { "sequence": 3586, "attempt_id": "8a174329-e922-4b3d-929c-707da8d2bd8d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.7274722260190174, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:08:57.421267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T12:08:57.421Z" }, { "sequence": 3587, "attempt_id": "a7dcc554-2dc3-42c0-a660-7c8e033561ca", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.706735857995227, "model_name": "claude-opus-5", "output_tokens": 131, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T12:08:57.674981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017525, "created_at": "2026-09-17T12:08:57.678Z" }, { "sequence": 3588, "attempt_id": "83c6afa5-31a3-41ca-9eb9-42cc1030363c", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7256684509338811, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:57.902737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T12:08:57.902Z" }, { "sequence": 3589, "attempt_id": "7cfc317a-dae5-4524-8db7-1dbfdfba30e3", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006388, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.564891543937847, "model_name": "claude-sonnet-5", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:58.142921+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006388, "created_at": "2026-09-17T12:08:58.144Z" }, { "sequence": 3590, "attempt_id": "273d83d7-5e3e-4421-a5ad-00f1b76429bd", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 5.548872505896725, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:08:58.376790+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T12:08:58.376Z" }, { "sequence": 3591, "attempt_id": "9c7fc2e5-1b4c-4e85-b0cc-f4a77f5f9740", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01587, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.703928622067906, "model_name": "claude-opus-5", "output_tokens": 73, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:58.595890+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01587, "created_at": "2026-09-17T12:08:58.596Z" }, { "sequence": 3592, "attempt_id": "a69b8f2b-b590-4add-94a2-73373d44366b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 4.0602688289945945, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:58.866577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T12:08:58.868Z" }, { "sequence": 3593, "attempt_id": "0b4c044c-7c8a-463d-ab5a-e1c12715c6af", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007218, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.841779852984473, "model_name": "claude-sonnet-5", "output_tokens": 135, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:59.342645+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007218, "created_at": "2026-09-17T12:08:59.342Z" }, { "sequence": 3594, "attempt_id": "2d843c45-0e96-47b7-8564-53fcbb3f1ae0", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.746270367060788, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:59.594610+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:08:59.594Z" }, { "sequence": 3595, "attempt_id": "632eba83-efbf-4ad5-9840-2dc2fb78cbdd", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01742, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.327764311921783, "model_name": "claude-opus-5", "output_tokens": 110, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:59.823338+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01742, "created_at": "2026-09-17T12:08:59.823Z" }, { "sequence": 3596, "attempt_id": "f08a9f4d-74f7-4b68-bc27-da8050d12106", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.244341160985641, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:00.379588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T12:09:00.379Z" }, { "sequence": 3597, "attempt_id": "8add293c-2eee-4849-9274-85e790490510", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.2586666180286556, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:00.629458+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T12:09:00.629Z" }, { "sequence": 3598, "attempt_id": "f1f89530-849b-47e0-b1c8-d82be58bb401", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.2020029140403494, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:00.859773+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:09:00.861Z" }, { "sequence": 3599, "attempt_id": "2819d12d-a8a8-4ca6-9e6d-9098fceffc49", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.496376744005829, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:01.080572+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T12:09:01.080Z" }, { "sequence": 3600, "attempt_id": "86c31aca-83c1-48f2-b07c-ea4ac56fdb89", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.0607266270089895, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:01.335234+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T12:09:01.336Z" }, { "sequence": 3601, "attempt_id": "00e43743-f4c9-4b13-810a-f36e440670ef", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.6845004160422832, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:01.558564+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T12:09:01.560Z" }, { "sequence": 3602, "attempt_id": "5dbe23d4-6411-4c91-88aa-d8b20b6690ca", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-021", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.03061, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 5.481913976022042, "model_name": "claude-fable-5-1", "output_tokens": 25, "prediction": { "choice": "D" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:01.779749+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03061, "created_at": "2026-09-17T12:09:01.779Z" }, { "sequence": 3603, "attempt_id": "c3d87ff0-2c81-4537-a4dc-ad8630578327", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6392772360704839, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:02.667200+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T12:09:02.667Z" }, { "sequence": 3604, "attempt_id": "26c1b3d8-ba72-4412-ab94-ba1d61a4a604", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.3536798630375415, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:03.029455+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T12:09:03.029Z" }, { "sequence": 3605, "attempt_id": "f86aeee6-88b8-4533-b2bb-5f5fd9c2c82a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006538, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.4679402239853516, "model_name": "claude-sonnet-5", "output_tokens": 91, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:03.244947+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006538, "created_at": "2026-09-17T12:09:03.247Z" }, { "sequence": 3606, "attempt_id": "f4b2947c-bd03-4a52-97d9-e1aafeff33a7", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01637, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 3.357606149977073, "model_name": "claude-opus-5", "output_tokens": 92, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:03.475559+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01637, "created_at": "2026-09-17T12:09:03.477Z" }, { "sequence": 3607, "attempt_id": "17b16d89-6cf6-4d89-ba33-7392bf3d0ed2", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.7422940699616447, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:04.218410+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:09:04.218Z" }, { "sequence": 3608, "attempt_id": "a1233e4a-d061-4c44-9dcf-ab700dee8c89", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 5.46878115599975, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:04.421577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T12:09:04.421Z" }, { "sequence": 3609, "attempt_id": "7ddc9d1b-24c8-4a32-9362-ab3e7f414ce6", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.1378943430026993, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:05.033205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T12:09:05.033Z" }, { "sequence": 3610, "attempt_id": "f71adcfd-afb7-4636-9e5c-165af38ea9b5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017575, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.874458491918631, "model_name": "claude-opus-5", "output_tokens": 113, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:05.276937+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.017575, "created_at": "2026-09-17T12:09:05.277Z" }, { "sequence": 3611, "attempt_id": "1f775781-7fe5-4081-8d75-d9bbaf9111e4", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.041031491942704, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:05.504208+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T12:09:05.506Z" }, { "sequence": 3612, "attempt_id": "1e766eaf-a25c-4799-9354-a1945289af52", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.2851100439438596, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:05.745627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T12:09:05.746Z" }, { "sequence": 3613, "attempt_id": "efc748bd-0e80-4b00-b750-4c5ab2f9a804", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00859, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 4.297187029966153, "model_name": "claude-sonnet-5", "output_tokens": 269, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:05.973129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00859, "created_at": "2026-09-17T12:09:05.973Z" }, { "sequence": 3614, "attempt_id": "6434b1c8-8cc0-48a9-a87e-94b35e25175d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 4.038416246068664, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:06.184254+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T12:09:06.184Z" }, { "sequence": 3615, "attempt_id": "5f57886c-3237-4780-b118-53bb101b1b83", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.2146584800211713, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:06.492022+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T12:09:06.492Z" }, { "sequence": 3616, "attempt_id": "a8510117-86f5-4199-835d-ec5701cf7b73", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006768, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.2529247290221974, "model_name": "claude-sonnet-5", "output_tokens": 109, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:06.894988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006768, "created_at": "2026-09-17T12:09:06.895Z" }, { "sequence": 3617, "attempt_id": "2d5846d4-51d4-4202-b85e-d62e7dec0b05", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01572, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.703425161074847, "model_name": "claude-opus-5", "output_tokens": 61, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:07.141222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01572, "created_at": "2026-09-17T12:09:07.141Z" }, { "sequence": 3618, "attempt_id": "69fb6a53-220a-4a4d-a919-5e7dfeffa329", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.628795061027631, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:07.384502+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T12:09:07.384Z" }, { "sequence": 3619, "attempt_id": "a4c5b8d9-1249-4d77-8b95-99929d03ae62", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.1723639840492979, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:07.647666+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T12:09:07.647Z" }, { "sequence": 3620, "attempt_id": "755f507c-0a22-48e2-818d-2ebaf2b676a5", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 7.6905029020272195, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:07.900353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T12:09:07.901Z" }, { "sequence": 3621, "attempt_id": "a4eef379-9956-44db-92ad-3be42f23d59f", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.2470681429840624, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:08.128372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T12:09:08.128Z" }, { "sequence": 3622, "attempt_id": "188337aa-8c86-492d-bf3b-a2c69b2d7f58", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.2151726440060884, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:08.365531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T12:09:08.367Z" }, { "sequence": 3623, "attempt_id": "1b695afc-8227-4dbb-a43a-f6efc828f335", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.1657958870055154, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:08.613167+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T12:09:08.614Z" }, { "sequence": 3624, "attempt_id": "9547d889-84be-4466-b0da-a98c9660bde5", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-003", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.6930869950447232, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:09.058939+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T12:09:09.059Z" }, { "sequence": 3625, "attempt_id": "5bb9dea1-9cdb-4087-bce8-fcb08cdb4bdb", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.030306814005598, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:09.537482+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T12:09:09.537Z" }, { "sequence": 3626, "attempt_id": "a9773ea8-1dc5-4fde-8c74-71a0605681fc", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006578, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.1329773040488362, "model_name": "claude-sonnet-5", "output_tokens": 90, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:10.274628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006578, "created_at": "2026-09-17T12:09:10.274Z" }, { "sequence": 3627, "attempt_id": "4e215740-2fa0-4213-a40a-ec4617e22f3c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01767, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.5862797929439694, "model_name": "claude-opus-5", "output_tokens": 139, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:10.463168+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01767, "created_at": "2026-09-17T12:09:10.463Z" }, { "sequence": 3628, "attempt_id": "4151f796-517d-459e-a5d7-2b5682326e3c", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridgaps-002", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001851, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.590163360000588, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:10.642108+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001851, "created_at": "2026-09-17T12:09:10.642Z" }, { "sequence": 3629, "attempt_id": "73fea990-1656-4dc2-a237-55b16be176bb", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015445, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.8830589200370014, "model_name": "claude-opus-5", "output_tokens": 56, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:10.866695+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015445, "created_at": "2026-09-17T12:09:10.868Z" }, { "sequence": 3630, "attempt_id": "82db7978-6906-4fbf-91d0-e09eb0f65307", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006048, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 3.250611466006376, "model_name": "claude-sonnet-5", "output_tokens": 43, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:11.379413+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006048, "created_at": "2026-09-17T12:09:11.379Z" }, { "sequence": 3631, "attempt_id": "b4c82927-8f02-40e3-bfb7-afe4c17c66a4", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.680386846070178, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:11.618838+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T12:09:11.621Z" }, { "sequence": 3632, "attempt_id": "44fd1fd6-acde-4fc3-9791-1edcaec32e52", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016105, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.0845181989716366, "model_name": "claude-opus-5", "output_tokens": 79, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:11.930177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016105, "created_at": "2026-09-17T12:09:11.930Z" }, { "sequence": 3633, "attempt_id": "cd7e1ac1-5e7f-4724-a0c0-1553184cd5d0", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006372, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.264330642996356, "model_name": "claude-sonnet-5", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:12.557697+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006372, "created_at": "2026-09-17T12:09:12.557Z" }, { "sequence": 3634, "attempt_id": "35132394-279c-4ef2-ba8f-9a42e61be033", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 1.512369236908853, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:12.753624+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T12:09:12.753Z" }, { "sequence": 3635, "attempt_id": "e5cd387c-37d8-46cb-8d3f-69a676da9146", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.070941030047834, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:12.939088+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00571, "created_at": "2026-09-17T12:09:12.939Z" }, { "sequence": 3636, "attempt_id": "0fb8bde7-8277-4bcf-aafe-626f25878a4a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 6.711406570975669, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:13.136218+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T12:09:13.136Z" }, { "sequence": 3637, "attempt_id": "d1f0f20c-7fcf-4f05-bef7-09933b58de95", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.7771751469699666, "model_name": "claude-opus-5", "output_tokens": 51, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:13.635250+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015275, "created_at": "2026-09-17T12:09:13.635Z" }, { "sequence": 3638, "attempt_id": "046fcb79-392e-4b6a-9b51-b431c9042c52", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-008", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 1.3117674749810249, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:14.065711+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T12:09:14.065Z" }, { "sequence": 3639, "attempt_id": "d8cb2625-5cde-4094-96bf-4778bfe2ad31", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9074537679553032, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:14.263425+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T12:09:14.263Z" }, { "sequence": 3640, "attempt_id": "17f2a1f3-4343-410e-91ae-67aaf35b7d34", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.1835320310201496, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:14.440568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014525, "created_at": "2026-09-17T12:09:14.440Z" }, { "sequence": 3641, "attempt_id": "bba6eaa8-7637-45cc-9f33-2b9fd8017f13", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02983, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.743830605992116, "model_name": "claude-fable-5-1", "output_tokens": 31, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": " {\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:14.630882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02983, "created_at": "2026-09-17T12:09:14.631Z" }, { "sequence": 3642, "attempt_id": "a7d654fa-78f6-4b41-ad82-add4751f6503", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6489058720180765, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:14.821221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T12:09:14.821Z" }, { "sequence": 3643, "attempt_id": "d93f45be-f1ff-40f4-8002-17e4f848d62a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02872, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.979646266088821, "model_name": "claude-fable-5-1", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:14.996983+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02872, "created_at": "2026-09-17T12:09:14.997Z" }, { "sequence": 3644, "attempt_id": "38a5a9f1-bb76-4ff4-a699-68608af8af49", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.1898191809887066, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:15.508636+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T12:09:15.508Z" }, { "sequence": 3645, "attempt_id": "325b7e71-dc35-4329-8bfa-583a0318f80e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.873793622944504, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:16.010350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014315, "created_at": "2026-09-17T12:09:16.010Z" }, { "sequence": 3646, "attempt_id": "58655d4f-164e-4baa-87d7-9fb799a3883b", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.538970006047748, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:16.229623+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T12:09:16.232Z" }, { "sequence": 3647, "attempt_id": "10f32b68-3ab9-45de-89cc-045b087ea5b7", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.191670508007519, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:16.448470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:09:16.448Z" }, { "sequence": 3648, "attempt_id": "d49b834c-9281-4d2b-b673-076c7900245f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 4.824583631940186, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:16.674439+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T12:09:16.674Z" }, { "sequence": 3649, "attempt_id": "b3f8f2f2-83cd-485f-8b49-32b4cd1a8d10", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.5904615969629958, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:16.936659+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T12:09:16.936Z" }, { "sequence": 3650, "attempt_id": "b2734b3f-e665-4775-aa52-c5ffc139950f", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.440799474949017, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:17.175833+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T12:09:17.176Z" }, { "sequence": 3651, "attempt_id": "52f53fa4-2e1a-431f-8ba5-93024aed194b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.25693367398344, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:17.461672+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014395, "created_at": "2026-09-17T12:09:17.461Z" }, { "sequence": 3652, "attempt_id": "1b10b72c-bc80-4372-9034-0aa39aa407c9", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.9855695770820603, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:17.679441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005758, "created_at": "2026-09-17T12:09:17.679Z" }, { "sequence": 3653, "attempt_id": "60160b95-e84d-41ef-add1-74453625509b", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-001", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.5488558019278571, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:17.864952+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T12:09:17.866Z" }, { "sequence": 3654, "attempt_id": "63b490e2-8a4b-4f88-a0da-80819b281b0b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.660805229912512, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:18.053809+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T12:09:18.055Z" }, { "sequence": 3655, "attempt_id": "df941a97-5ad5-455b-8fff-38b030be13ad", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 5.104505092953332, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:18.246353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T12:09:18.246Z" }, { "sequence": 3656, "attempt_id": "4f86f2c6-1483-488a-8618-ffdac5300c23", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00635, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.4031200320459902, "model_name": "claude-sonnet-5", "output_tokens": 65, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:19.340228+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00635, "created_at": "2026-09-17T12:09:19.340Z" }, { "sequence": 3657, "attempt_id": "9f47834f-8f62-45ca-8431-c72ab1715763", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.3179214539704844, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "E" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T12:09:19.526335+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T12:09:19.526Z" }, { "sequence": 3658, "attempt_id": "3150c65f-582a-4883-82f0-0e75913145f8", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01655, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.0718910479918122, "model_name": "claude-opus-5", "output_tokens": 92, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T12:09:20.001201+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01655, "created_at": "2026-09-17T12:09:20.001Z" }, { "sequence": 3659, "attempt_id": "3c1d71d7-a8dc-4109-9b7b-0438c3adcb49", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.7340721199288964, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:20.446630+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001862, "created_at": "2026-09-17T12:09:20.446Z" }, { "sequence": 3660, "attempt_id": "b78d1ad7-41da-44a6-828c-9b292e379766", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 4.27970632200595, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:20.728565+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T12:09:20.728Z" }, { "sequence": 3661, "attempt_id": "5407cb7b-4436-47bf-bed0-f9b5a72b072f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 6.207112283911556, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:21.405667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T12:09:21.405Z" }, { "sequence": 3662, "attempt_id": "cfa57141-104c-4b8f-b026-e39e55d03dcf", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01652, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 3.251068320008926, "model_name": "claude-opus-5", "output_tokens": 96, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:21.688255+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01652, "created_at": "2026-09-17T12:09:21.688Z" }, { "sequence": 3663, "attempt_id": "d5c66013-905b-45f0-b168-7dbb5e3ff1f5", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.5690809070365503, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:21.999975+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02881, "created_at": "2026-09-17T12:09:22.000Z" }, { "sequence": 3664, "attempt_id": "fd9a5d56-1ca8-4baa-b462-9c540712e8a2", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006228, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.555660932091996, "model_name": "claude-sonnet-5", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:22.189290+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006228, "created_at": "2026-09-17T12:09:22.190Z" }, { "sequence": 3665, "attempt_id": "7592847c-348d-4ed0-9760-ac2a7582f284", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008228, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 4.403347961953841, "model_name": "claude-sonnet-5", "output_tokens": 236, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T12:09:22.390776+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008228, "created_at": "2026-09-17T12:09:22.392Z" }, { "sequence": 3666, "attempt_id": "0ba176d7-1e34-4b8c-aea6-a8dd754064a4", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 1.205817612935789, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:22.809891+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001833, "created_at": "2026-09-17T12:09:22.810Z" }, { "sequence": 3667, "attempt_id": "0d29dac3-892a-4caa-a110-4e68d2a8ca44", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 2.40151453204453, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:23.039650+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01421, "created_at": "2026-09-17T12:09:23.039Z" }, { "sequence": 3668, "attempt_id": "0522def3-8fee-4915-a524-d82bc95bb13a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 2.222823846968822, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:23.281594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005684, "created_at": "2026-09-17T12:09:23.281Z" }, { "sequence": 3669, "attempt_id": "b0447cba-406a-48ef-b475-281e62624ea8", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.7456695429282263, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:23.485506+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001876, "created_at": "2026-09-17T12:09:23.485Z" }, { "sequence": 3670, "attempt_id": "029c40c7-106d-4ddf-93df-3589a3e97c9a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 5.972971174051054, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T12:09:23.672713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T12:09:23.672Z" }, { "sequence": 3671, "attempt_id": "f8b6a532-8e66-4f75-af20-381e43bbcca8", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02292, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 6.009647022001445, "model_name": "claude-opus-5", "output_tokens": 330, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T12:09:23.879657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02292, "created_at": "2026-09-17T12:09:23.880Z" }, { "sequence": 3672, "attempt_id": "bc52bed5-67c8-40c5-8d79-2fad060e657c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 1.565758976037614, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:24.119232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005798, "created_at": "2026-09-17T12:09:24.120Z" }, { "sequence": 3673, "attempt_id": "629bb7e2-df7e-4459-86e4-49189898928c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 4.386352004017681, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:24.588984+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02844, "created_at": "2026-09-17T12:09:24.589Z" }, { "sequence": 3674, "attempt_id": "46f12893-2ce3-4f0f-9d0b-42b258bfd0b4", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 1.2576991930836812, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:25.130326+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T12:09:25.130Z" }, { "sequence": 3675, "attempt_id": "00326f0c-fbb4-4a8d-b0be-bcb59ed79ca3", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.027294861036353, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:25.388167+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T12:09:25.388Z" }, { "sequence": 3676, "attempt_id": "9c3cc2c2-bbb9-4f18-a88b-682c4af6e135", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.8905104349832982, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:25.614237+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T12:09:25.614Z" }, { "sequence": 3677, "attempt_id": "dad59a82-8f1d-489e-b495-55fdcf24b03e", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016345, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.5524460420710966, "model_name": "claude-opus-5", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:25.838261+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016345, "created_at": "2026-09-17T12:09:25.838Z" }, { "sequence": 3678, "attempt_id": "d9d5a18a-6102-4ffb-958b-772719dc3d5a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 1.2017014189623296, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:26.049509+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001831, "created_at": "2026-09-17T12:09:26.051Z" }, { "sequence": 3679, "attempt_id": "1e16f3d1-a7b0-47db-ad0b-55168604e119", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.738939507980831, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:26.252934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005676, "created_at": "2026-09-17T12:09:26.254Z" }, { "sequence": 3680, "attempt_id": "9eb91206-836f-4787-8d1d-226b9c461fee", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.751092229038477, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:26.787020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T12:09:26.787Z" }, { "sequence": 3681, "attempt_id": "a3af3d34-28a4-4370-847d-22decd25c526", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.6977497891057283, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:27.033801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01419, "created_at": "2026-09-17T12:09:27.036Z" }, { "sequence": 3682, "attempt_id": "2c00951e-7266-4f85-bbb1-d473835c7635", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.8001180819701403, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:27.273428+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T12:09:27.273Z" }, { "sequence": 3683, "attempt_id": "650637d6-d149-4a85-b24f-4f0df637dcc8", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.261525260983035, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:27.517262+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0284, "created_at": "2026-09-17T12:09:27.519Z" }, { "sequence": 3684, "attempt_id": "ee9ddcf7-9de4-4a92-8686-709745c8f436", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8969414910534397, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:27.745694+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T12:09:27.745Z" }, { "sequence": 3685, "attempt_id": "44e829d1-17c8-4fda-8e66-294cbb103e15", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6711723370244727, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:28.181444+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:09:28.181Z" }, { "sequence": 3686, "attempt_id": "035a1816-c4da-4bde-b299-0446637b71d5", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.3728753379546106, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:28.380252+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T12:09:28.382Z" }, { "sequence": 3687, "attempt_id": "9a796da7-be0f-4811-9ab9-80328c22e93c", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.3726321069989353, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:28.824420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T12:09:28.824Z" }, { "sequence": 3688, "attempt_id": "d1b0826a-568f-4533-933c-6311bc15610d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 7.033768536988646, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:29.039854+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02901, "created_at": "2026-09-17T12:09:29.040Z" }, { "sequence": 3689, "attempt_id": "4b729837-3cba-4b1c-aa93-0f2f3a295e00", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.010983929038048, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:29.255982+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T12:09:29.258Z" }, { "sequence": 3690, "attempt_id": "92f1a972-0fac-4a36-896c-d6824f0169d5", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 1.1421308839926496, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:29.522634+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001832, "created_at": "2026-09-17T12:09:29.522Z" }, { "sequence": 3691, "attempt_id": "1c8adff1-942c-405a-8570-4f23f3a3fec1", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.8292600100394338, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:29.776518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005666, "created_at": "2026-09-17T12:09:29.776Z" }, { "sequence": 3692, "attempt_id": "00614acb-92cd-434e-9e15-d9efe76f08a8", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.552591860992834, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:30.010170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T12:09:30.010Z" }, { "sequence": 3693, "attempt_id": "637d702b-8094-40dc-9657-75da5a2b8401", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.5298772959504277, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:30.243611+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T12:09:30.245Z" }, { "sequence": 3694, "attempt_id": "0cccbbd7-30e3-489c-9f21-07b7351b3457", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.3821549279382452, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:30.484250+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014165, "created_at": "2026-09-17T12:09:30.484Z" }, { "sequence": 3695, "attempt_id": "25c3515c-7674-417a-803b-f7f015338ae7", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.895375888911076, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:30.724853+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T12:09:30.725Z" }, { "sequence": 3696, "attempt_id": "447c5b63-d7a3-47ca-9512-e9084c7796c3", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 4.350167638040148, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:31.867780+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02835, "created_at": "2026-09-17T12:09:31.867Z" }, { "sequence": 3697, "attempt_id": "94112b63-410a-44c1-962f-5d4a436890e5", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-016", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.210386881022714, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:32.118274+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T12:09:32.120Z" }, { "sequence": 3698, "attempt_id": "c6da9ade-6013-47e9-b2f5-deb6a5c4d5cf", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.58785557304509, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:32.358131+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T12:09:32.359Z" }, { "sequence": 3699, "attempt_id": "8000a969-1257-4cfb-ac85-881195eb91a6", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01677, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.294167770072818, "model_name": "claude-opus-5", "output_tokens": 103, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:32.585043+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01677, "created_at": "2026-09-17T12:09:32.585Z" }, { "sequence": 3700, "attempt_id": "897ed765-9a93-4ccf-9ceb-85ee3807be21", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006838, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.145954924984835, "model_name": "claude-sonnet-5", "output_tokens": 116, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:32.824564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006838, "created_at": "2026-09-17T12:09:32.824Z" }, { "sequence": 3701, "attempt_id": "51b66df3-f8ea-4348-9e0d-66f99efc4a4a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6317133799893782, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:33.085598+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:09:33.085Z" }, { "sequence": 3702, "attempt_id": "49116c95-5c11-4b3d-a5d2-9e6b7cac9910", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.468582038069144, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:33.426223+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T12:09:33.426Z" }, { "sequence": 3703, "attempt_id": "5116c71b-f763-46f4-8a5b-c1ffd3f17577", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007898, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.3312806920148432, "model_name": "claude-sonnet-5", "output_tokens": 203, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:33.674307+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007898, "created_at": "2026-09-17T12:09:33.676Z" }, { "sequence": 3704, "attempt_id": "4d88facd-23ca-4c44-92bb-8fcc002cc3a0", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.028682340984233, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:33.934225+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T12:09:33.939Z" }, { "sequence": 3705, "attempt_id": "bb007ec2-70cc-47d8-85c7-e619b6d34edd", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.5874728410271928, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:34.168684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T12:09:34.168Z" }, { "sequence": 3706, "attempt_id": "04ea5be0-187a-438f-ac4c-85d29ff0fefd", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.4985559859778732, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:34.822755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T12:09:34.822Z" }, { "sequence": 3707, "attempt_id": "612a5bd8-976f-4ae2-ba7e-31439e24ad13", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.2720221279887483, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:35.096912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01447, "created_at": "2026-09-17T12:09:35.097Z" }, { "sequence": 3708, "attempt_id": "40d282fe-346e-42c9-9ca7-bee67ca70c9d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02002, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.244075958034955, "model_name": "claude-opus-5", "output_tokens": 214, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:35.334268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02002, "created_at": "2026-09-17T12:09:35.335Z" }, { "sequence": 3709, "attempt_id": "902d0c9d-118f-49a3-9eec-e3f7ea853a76", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006478, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.3372013369807974, "model_name": "claude-sonnet-5", "output_tokens": 80, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:35.579547+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006478, "created_at": "2026-09-17T12:09:35.579Z" }, { "sequence": 3710, "attempt_id": "f97af48b-37fb-4901-9af4-7815e8fad9ef", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-002", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001928, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.16552681196481, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "A" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:35.842616+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001928, "created_at": "2026-09-17T12:09:35.844Z" }, { "sequence": 3711, "attempt_id": "32324c1b-801f-4edf-b22f-af5a352e482a", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.670571875059977, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:36.487999+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T12:09:36.488Z" }, { "sequence": 3712, "attempt_id": "71d99db0-307b-460a-b0c6-4c5f22b4d7ca", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-015", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.283471809932962, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:37.126474+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T12:09:37.127Z" }, { "sequence": 3713, "attempt_id": "1ad90da7-3ebe-4978-ba79-113665ee3b7e", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.681650090031326, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T12:09:37.369844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T12:09:37.370Z" }, { "sequence": 3714, "attempt_id": "c321d115-87ab-42ef-9bd2-257d1090e17b", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 7.295461642905138, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:38.020713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T12:09:38.020Z" }, { "sequence": 3715, "attempt_id": "ee2468e5-0249-42fb-bf27-b7c45c116b3f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6547806380549446, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:38.263225+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:09:38.267Z" }, { "sequence": 3716, "attempt_id": "d5f0650e-7a0e-4393-851b-4b860c4dde3f", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01672, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.107550722081214, "model_name": "claude-opus-5", "output_tokens": 82, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:38.503798+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01672, "created_at": "2026-09-17T12:09:38.504Z" }, { "sequence": 3717, "attempt_id": "6c3705b3-603b-4c6a-96ff-de52c6a7a825", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6365681189345196, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:39.006785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T12:09:39.006Z" }, { "sequence": 3718, "attempt_id": "7d4c33c7-2c75-4db3-8176-8a95de8f9d35", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.3954857600620016, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:39.254338+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T12:09:39.256Z" }, { "sequence": 3719, "attempt_id": "c53b7dd4-3987-4c22-ab3a-90678269a280", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.019945, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.137193478061818, "model_name": "claude-opus-5", "output_tokens": 211, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T12:09:39.481222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.019945, "created_at": "2026-09-17T12:09:39.481Z" }, { "sequence": 3720, "attempt_id": "4b29aa5e-658b-4bd7-b1f0-dfe34d2ffece", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.494633733993396, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:39.741190+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T12:09:39.742Z" }, { "sequence": 3721, "attempt_id": "fdd95ea1-8114-49b0-85dd-c3e88677b8fa", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007388, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.7955728280358016, "model_name": "claude-sonnet-5", "output_tokens": 152, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:39.969403+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007388, "created_at": "2026-09-17T12:09:39.969Z" }, { "sequence": 3722, "attempt_id": "dc784b5d-bb86-4c39-b0af-0176e0bf46bb", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.5370337018975988, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:40.201008+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001834, "created_at": "2026-09-17T12:09:40.201Z" }, { "sequence": 3723, "attempt_id": "a91c24ac-62b1-427f-92bb-95eccf628bbb", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8197744010249153, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:40.558396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005682, "created_at": "2026-09-17T12:09:40.558Z" }, { "sequence": 3724, "attempt_id": "69dbdefc-9a17-4458-9f84-8df4ac9ab419", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.45995413698256, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:40.964187+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014205, "created_at": "2026-09-17T12:09:40.964Z" }, { "sequence": 3725, "attempt_id": "9b43d4f0-9402-4ac2-8783-8763339b2e10", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 2.851811353932135, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:41.207215+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02843, "created_at": "2026-09-17T12:09:41.207Z" }, { "sequence": 3726, "attempt_id": "49365cf0-8e3d-4e48-8114-2f9a292e1028", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 1.2426200719783083, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:41.463580+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T12:09:41.463Z" }, { "sequence": 3727, "attempt_id": "e116b981-8897-469d-880f-0ff77b1258ac", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6295738230692223, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:42.093611+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001858, "created_at": "2026-09-17T12:09:42.093Z" }, { "sequence": 3728, "attempt_id": "0e32042f-eeeb-4fae-84f6-cd65366c7324", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006348, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.8254505830118433, "model_name": "claude-sonnet-5", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:43.019307+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006348, "created_at": "2026-09-17T12:09:43.019Z" }, { "sequence": 3729, "attempt_id": "988458c4-695f-40df-93c1-b57b893fd3be", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015545, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 3.3209629940101877, "model_name": "claude-opus-5", "output_tokens": 59, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:43.263107+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015545, "created_at": "2026-09-17T12:09:43.264Z" }, { "sequence": 3730, "attempt_id": "622809b8-e10d-46b6-9b17-83bf94a339e3", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.8545936199370772, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:43.531299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005726, "created_at": "2026-09-17T12:09:43.531Z" }, { "sequence": 3731, "attempt_id": "b470d442-b0e3-48aa-994e-65b257a1d153", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.519625935005024, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:43.776341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014315, "created_at": "2026-09-17T12:09:43.777Z" }, { "sequence": 3732, "attempt_id": "ffa2f578-1504-4e1a-9f4b-a89ddf7bd066", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 4.457491965033114, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:44.191601+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T12:09:44.191Z" }, { "sequence": 3733, "attempt_id": "be0b6439-e86b-470d-acb7-efb42be223b3", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.9298677350161597, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:44.449628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02865, "created_at": "2026-09-17T12:09:44.449Z" }, { "sequence": 3734, "attempt_id": "8de28c8e-8b48-47dc-95f2-1471594c7228", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.2592387000331655, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T12:09:44.961639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02991, "created_at": "2026-09-17T12:09:44.961Z" }, { "sequence": 3735, "attempt_id": "e2eaaf8b-7331-46ce-bff8-e77d78bbad16", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016795, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.8407287509180605, "model_name": "claude-opus-5", "output_tokens": 85, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T12:09:45.184836+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016795, "created_at": "2026-09-17T12:09:45.185Z" }, { "sequence": 3736, "attempt_id": "4586b074-ab67-4636-9bfd-bedea500e0a4", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-distribution-013", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.2004007059149444, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:45.380284+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001933, "created_at": "2026-09-17T12:09:45.380Z" }, { "sequence": 3737, "attempt_id": "89115849-4a5a-4390-a7e8-ef439f67cdd9", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006528, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.3348386849975213, "model_name": "claude-sonnet-5", "output_tokens": 66, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T12:09:45.598262+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006528, "created_at": "2026-09-17T12:09:45.598Z" }, { "sequence": 3738, "attempt_id": "6eb00371-6699-493b-aa14-5d9dbc4bc286", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-wrapalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 1.0365102919749916, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:45.836872+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001854, "created_at": "2026-09-17T12:09:45.837Z" }, { "sequence": 3739, "attempt_id": "e7b8b961-f114-4537-bf67-1c5e79f8e53f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.4663895219564438, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:46.303526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:09:46.303Z" }, { "sequence": 3740, "attempt_id": "d86f6f79-aab2-4d4d-91c5-f5dc272ea0f7", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 10.603448732057586, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:46.689445+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T12:09:46.689Z" }, { "sequence": 3741, "attempt_id": "214f5ba2-5c12-4f60-946a-18b57d2b6482", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016445, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.8408228179905564, "model_name": "claude-opus-5", "output_tokens": 95, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:46.930291+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016445, "created_at": "2026-09-17T12:09:46.932Z" }, { "sequence": 3742, "attempt_id": "35b64d89-aa53-4445-b102-b321466a29f4", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.750715372036211, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:47.527403+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02871, "created_at": "2026-09-17T12:09:47.527Z" }, { "sequence": 3743, "attempt_id": "ed6b9c1d-9a26-431b-9c21-ee5c28b971b9", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007168, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 3.30020812607836, "model_name": "claude-sonnet-5", "output_tokens": 154, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:47.812319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007168, "created_at": "2026-09-17T12:09:47.818Z" }, { "sequence": 3744, "attempt_id": "49945dda-b813-4b30-9014-9c34f2eaa547", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.7813130930298939, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:48.053609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001852, "created_at": "2026-09-17T12:09:48.055Z" }, { "sequence": 3745, "attempt_id": "4750cfd5-9258-46ee-9f6a-8a641d1a4ad7", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016638, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 13.729338833945803, "model_name": "claude-sonnet-5", "output_tokens": 1077, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T12:09:48.293742+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016638, "created_at": "2026-09-17T12:09:48.293Z" }, { "sequence": 3746, "attempt_id": "25633b96-0f3f-40f8-9ddf-41034093e81b", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.4945498630404472, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:48.535772+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T12:09:48.535Z" }, { "sequence": 3747, "attempt_id": "88ed5c92-bb7a-4d43-977f-c50b3fb89e21", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.577894364017993, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:48.789291+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T12:09:48.792Z" }, { "sequence": 3748, "attempt_id": "e869558e-cdc5-4be4-afb0-2e8fa47bbe24", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.5639592779334635, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:49.140200+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T12:09:49.140Z" }, { "sequence": 3749, "attempt_id": "a002508a-6e4c-4707-96c0-e3273487ef6f", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.579432393074967, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:49.667721+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02866, "created_at": "2026-09-17T12:09:49.667Z" }, { "sequence": 3750, "attempt_id": "50f85f42-bf9c-4c37-9855-849fe59287d6", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gapcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006348, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.835161685012281, "model_name": "claude-sonnet-5", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:49.907886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006348, "created_at": "2026-09-17T12:09:49.909Z" }, { "sequence": 3751, "attempt_id": "74f77e63-eec7-4cd3-87be-b28df6ab4c12", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.6547564569627866, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:50.151065+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00571, "created_at": "2026-09-17T12:09:50.151Z" }, { "sequence": 3752, "attempt_id": "46fe6c7e-4f5b-48dc-beec-235407cd62d4", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 1.2955318640451878, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:50.396837+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T12:09:50.397Z" }, { "sequence": 3753, "attempt_id": "bf383f1f-96c3-44a7-9fc7-5fbbb62460fb", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.4178943829610944, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:50.653470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014275, "created_at": "2026-09-17T12:09:50.653Z" }, { "sequence": 3754, "attempt_id": "f8f907b7-1a2e-4612-859c-f781522ebd71", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-gridspan-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 1.1191340850200504, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:51.027332+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001859, "created_at": "2026-09-17T12:09:51.027Z" }, { "sequence": 3755, "attempt_id": "42a94df5-87ae-4563-abc6-af84e8a4b3e7", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.223421359900385, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:51.287639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01434, "created_at": "2026-09-17T12:09:51.289Z" }, { "sequence": 3756, "attempt_id": "8878a0b3-bc11-4607-ab3f-6b9527c111b6", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.1331924910191447, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:52.032496+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001866, "created_at": "2026-09-17T12:09:52.032Z" }, { "sequence": 3757, "attempt_id": "ed66e3bc-fb57-4ded-a6ab-f4822cde45af", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 4.3933457439998165, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:52.279711+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02857, "created_at": "2026-09-17T12:09:52.283Z" }, { "sequence": 3758, "attempt_id": "6a003c93-81a9-4cfa-9a7e-b74c784b60c7", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.416067873011343, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:52.514717+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0287, "created_at": "2026-09-17T12:09:52.514Z" }, { "sequence": 3759, "attempt_id": "b0746bdd-2abf-448d-9f95-9c48684e74b2", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-gapcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01967, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 6.0006373500218615, "model_name": "claude-opus-5", "output_tokens": 225, "prediction": { "choice": "A" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:52.774199+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01967, "created_at": "2026-09-17T12:09:52.774Z" }, { "sequence": 3760, "attempt_id": "0e509b79-dd52-406d-8b0a-2069f1707154", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006888, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.5750955160474405, "model_name": "claude-sonnet-5", "output_tokens": 121, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:53.229047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006888, "created_at": "2026-09-17T12:09:53.229Z" }, { "sequence": 3761, "attempt_id": "0765a1d0-71cf-470c-8f75-588416fe9127", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006276, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 3.9754105739993975, "model_name": "claude-sonnet-5", "output_tokens": 65, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:53.471773+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006276, "created_at": "2026-09-17T12:09:53.472Z" }, { "sequence": 3762, "attempt_id": "df4b8e02-202a-4720-af05-240c3d35d309", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.416442007990554, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:53.734551+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T12:09:53.738Z" }, { "sequence": 3763, "attempt_id": "eb3e4fdd-9d60-48e5-b860-5990c4aedf28", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.7872187339235097, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:53.969648+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:09:53.969Z" }, { "sequence": 3764, "attempt_id": "3b4ff662-ff52-4caf-b430-73787dcbf797", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.532094552065246, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:54.190190+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T12:09:54.190Z" }, { "sequence": 3765, "attempt_id": "c9ac72ef-1cc1-45c2-9090-9ade0eaabbea", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.8939761959481984, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:54.390204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02896, "created_at": "2026-09-17T12:09:54.390Z" }, { "sequence": 3766, "attempt_id": "80a9d259-09dd-40be-8619-bcb1866cb01a", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.608709970023483, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:54.588436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:09:54.588Z" }, { "sequence": 3767, "attempt_id": "5a239266-766e-4b6b-928f-d565326c3663", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.3847472199704498, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:54.856851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005762, "created_at": "2026-09-17T12:09:54.857Z" }, { "sequence": 3768, "attempt_id": "5ea67467-802e-4b52-a1f4-14f1c0fb7d6f", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-004", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.6376529389526695, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:55.226416+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T12:09:55.226Z" }, { "sequence": 3769, "attempt_id": "3c5a7d27-5b06-4afc-8d3d-e4a28b8ff083", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.333905070903711, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:55.469112+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014405, "created_at": "2026-09-17T12:09:55.469Z" }, { "sequence": 3770, "attempt_id": "4531dd9a-ec80-41b7-ae02-41d3e80282bb", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01617, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 5.243524444056675, "model_name": "claude-opus-5", "output_tokens": 79, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:55.886444+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01617, "created_at": "2026-09-17T12:09:55.886Z" }, { "sequence": 3771, "attempt_id": "72fc1425-d064-4fab-999f-aa3d907772e3", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.1571320169605315, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:56.146978+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02883, "created_at": "2026-09-17T12:09:56.148Z" }, { "sequence": 3772, "attempt_id": "edf44851-64e5-495a-831c-1c894b33efd9", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.63056990201585, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:56.394000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001855, "created_at": "2026-09-17T12:09:56.395Z" }, { "sequence": 3773, "attempt_id": "65f2b1c2-0730-413e-b912-59021df46161", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.964409779990092, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:57.072568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014375, "created_at": "2026-09-17T12:09:57.072Z" }, { "sequence": 3774, "attempt_id": "927f63f4-3fd6-496d-9a29-2ceb4265b024", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7343878019601107, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:57.334289+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00575, "created_at": "2026-09-17T12:09:57.335Z" }, { "sequence": 3775, "attempt_id": "70b0bb6b-70cd-4238-9d58-648837229856", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00618, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.8572990730172023, "model_name": "claude-sonnet-5", "output_tokens": 48, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:57.576340+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00618, "created_at": "2026-09-17T12:09:57.576Z" }, { "sequence": 3776, "attempt_id": "7a36bc74-26e3-4bf3-aa1f-7b150ffeb536", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 6.368188295979053, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:57.820797+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T12:09:57.822Z" }, { "sequence": 3777, "attempt_id": "a59dd924-6ce4-47b4-a14a-8f516228825d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.3958913860842586, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:58.060457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T12:09:58.060Z" }, { "sequence": 3778, "attempt_id": "b788d835-4f88-499e-8b3f-3b9f381ec04c", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.535837634000927, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:58.348683+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02877, "created_at": "2026-09-17T12:09:58.348Z" }, { "sequence": 3779, "attempt_id": "ccad6b6e-3a89-4425-9ba8-40d0fb2363d7", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016625, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.663676081923768, "model_name": "claude-opus-5", "output_tokens": 95, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T12:09:58.592668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016625, "created_at": "2026-09-17T12:09:58.592Z" }, { "sequence": 3780, "attempt_id": "e5fa4191-afc4-43a8-81a6-7f1645d37c7c", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.009283086983487, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:58.862846+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00571, "created_at": "2026-09-17T12:09:58.863Z" }, { "sequence": 3781, "attempt_id": "817e125f-146d-4e53-8733-21252a1dd7e4", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.0288546279771253, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:59.109366+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.014275, "created_at": "2026-09-17T12:09:59.109Z" }, { "sequence": 3782, "attempt_id": "da2b06ad-a579-4559-815c-d431f3c5547d", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 1.2322405689628795, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:59.341459+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001847, "created_at": "2026-09-17T12:09:59.341Z" }, { "sequence": 3783, "attempt_id": "b10bb794-4577-4870-a49d-f3164c1556b9", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.7490804670378566, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:59.598089+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001876, "created_at": "2026-09-17T12:09:59.598Z" }, { "sequence": 3784, "attempt_id": "d816157c-d7a7-48f0-902e-c432c5eef9df", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.6987355779856443, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:59.845997+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02857, "created_at": "2026-09-17T12:09:59.846Z" }, { "sequence": 3785, "attempt_id": "3f0efe41-b9f3-4f00-9a46-683736f3af13", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6084565379424021, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:10:00.200210+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00193, "created_at": "2026-09-17T12:10:00.200Z" }, { "sequence": 3786, "attempt_id": "3911542e-f51a-42d4-b0e4-855302164f3d", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015745, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.739813438965939, "model_name": "claude-opus-5", "output_tokens": 61, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:10:00.560944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015745, "created_at": "2026-09-17T12:10:00.561Z" }, { "sequence": 3787, "attempt_id": "88bd0c5f-508a-4f5c-8572-1458c075f521", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-textalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.5572165860794485, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:10:00.951268+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001887, "created_at": "2026-09-17T12:10:00.951Z" }, { "sequence": 3788, "attempt_id": "63a18950-5603-49ed-a8ee-eab838a71c2a", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.6435424860101193, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:10:01.151396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.005798, "created_at": "2026-09-17T12:10:01.151Z" }, { "sequence": 3789, "attempt_id": "3e7e602f-c3e5-4f88-b2f6-21d5ce82666a", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.3569706969428807, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:10:01.369699+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.015025, "created_at": "2026-09-17T12:10:01.371Z" }, { "sequence": 3790, "attempt_id": "d44136a8-7a4c-4682-8119-25c7c4497e35", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 3.9468813539715484, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:10:01.580665+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02901, "created_at": "2026-09-17T12:10:01.580Z" }, { "sequence": 3791, "attempt_id": "5159b372-c4b6-49bb-ae85-6d76ba019a3f", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.3900387639878318, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:10:01.974346+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00601, "created_at": "2026-09-17T12:10:01.974Z" }, { "sequence": 3792, "attempt_id": "7c269bdb-8581-4236-b6b7-305ac53c76bb", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-textalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.464632790070027, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:10:02.503320+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00581, "created_at": "2026-09-17T12:10:02.503Z" }, { "sequence": 3793, "attempt_id": "2202eec1-e4fb-4506-907b-9b5e7880fad2", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-textalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016625, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.900702119106427, "model_name": "claude-opus-5", "output_tokens": 95, "prediction": { "choice": "A" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:10:02.707797+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.016625, "created_at": "2026-09-17T12:10:02.707Z" }, { "sequence": 3794, "attempt_id": "4ee403f2-55da-4fac-aa98-537ebdb27e26", "run_id": "qualitative-20260917", "model_id": "claude-haiku-4-5-20251001", "task_id": "layoutbench-padcompare-006", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001879, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 1.4299144460819662, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "B" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:10:03.010920+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.001879, "created_at": "2026-09-17T12:10:03.011Z" }, { "sequence": 3795, "attempt_id": "1bc89c7b-cbe4-44f1-abb5-95b5c7fffc5d", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-textalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.475862516905181, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:10:03.204282+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02907, "created_at": "2026-09-17T12:10:03.204Z" }, { "sequence": 3796, "attempt_id": "093d2600-a568-443a-95eb-da8e8d73e335", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.404964267043397, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:10:03.394417+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.03007, "created_at": "2026-09-17T12:10:03.394Z" }, { "sequence": 3797, "attempt_id": "32990a3f-30df-47e1-9645-e77129e3f0d8", "run_id": "qualitative-20260917", "model_id": "claude-opus-5", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01724, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.2371656299801543, "model_name": "claude-opus-5", "output_tokens": 116, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:10:04.388897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01724, "created_at": "2026-09-17T12:10:04.389Z" }, { "sequence": 3798, "attempt_id": "9c2a6ece-b99f-4dc3-89bc-9221697d5ef6", "run_id": "qualitative-20260917", "model_id": "claude-sonnet-5", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006926, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.174259925959632, "model_name": "claude-sonnet-5", "output_tokens": 119, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:10:04.580142+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006926, "created_at": "2026-09-17T12:10:04.580Z" }, { "sequence": 3799, "attempt_id": "96d2eb98-429f-4c87-b711-49346b351210", "run_id": "qualitative-20260917", "model_id": "claude-fable-5-1", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02995, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 5.5445699649862945, "model_name": "claude-fable-5-1", "output_tokens": 25, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:10:06.312755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.02995, "created_at": "2026-09-17T12:10:06.312Z" } ], "specimens": [ { "task_id": "layoutbench-direction-001", "family": "direction", "group_id": "direction-row-light-0", "image_filename": "direction-row-light-0.png", "image_sha256": "4727c71ba70b443edf13136f7c7219a32ba1e65babccee09327f3918fee6ebf6", "ground_truth": { "choice": "B" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Stacked in one vertical column\nB: Side by side in one horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "column", "row" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-002", "family": "direction", "group_id": "direction-row-light-1", "image_filename": "direction-row-light-1.png", "image_sha256": "fd32b6baf1afbd4af974ea300b59601ffa3f2b26ef50baef8dc56cf1351752c1", "ground_truth": { "choice": "A" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Side by side in one horizontal row\nB: Stacked in one vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-003", "family": "direction", "group_id": "direction-row-dark-0", "image_filename": "direction-row-dark-0.png", "image_sha256": "e1054056f7333f3d07d4db078449b382741a50c1967ea87d3873042aeedd21b1", "ground_truth": { "choice": "B" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Stacked in one vertical column\nB: Side by side in one horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "column", "row" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-004", "family": "direction", "group_id": "direction-row-dark-1", "image_filename": "direction-row-dark-1.png", "image_sha256": "5d5d514272958fc67e30dfb635e8303dfec36af16d578450c675d558b918289a", "ground_truth": { "choice": "A" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Side by side in one horizontal row\nB: Stacked in one vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-005", "family": "direction", "group_id": "direction-column-light-0", "image_filename": "direction-column-light-0.png", "image_sha256": "d21144267d483280335db6abe4118b89b6b54f3abdc03c4fc9642a782294cfe8", "ground_truth": { "choice": "B" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Side by side in one horizontal row\nB: Stacked in one vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "row", "column" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-006", "family": "direction", "group_id": "direction-column-light-1", "image_filename": "direction-column-light-1.png", "image_sha256": "39fb86e81eb182c6ed20beacb763e3fd5b3fb3b4a2791e22d8d44c41725de7a3", "ground_truth": { "choice": "A" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Stacked in one vertical column\nB: Side by side in one horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "column", "row" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-007", "family": "direction", "group_id": "direction-column-dark-0", "image_filename": "direction-column-dark-0.png", "image_sha256": "f6d975c91e1cfa5205f4bd29576f5a82451dd51773c94d735d4a878702c812e2", "ground_truth": { "choice": "A" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Stacked in one vertical column\nB: Side by side in one horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "column", "row" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-008", "family": "direction", "group_id": "direction-column-dark-1", "image_filename": "direction-column-dark-1.png", "image_sha256": "1666a5a779971149d3acca75b3f20edfab2f6ca286e8f7d76d53825ceb8001a7", "ground_truth": { "choice": "B" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Side by side in one horizontal row\nB: Stacked in one vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-distribution-001", "family": "distribution", "group_id": "distribution-start-light-0", "image_filename": "distribution-start-light-0.png", "image_sha256": "6934f78a7a7500fa45f29d87d1be1c389ca15393173d7b916a6b76e77434b623", "ground_truth": { "choice": "D" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: All interior gaps and both outer spaces are equal\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: Packed at the start, touching that inner edge\nE: Equal outer spaces, each half the size of an interior gap\nF: Packed together in the middle, with much larger equal outer spaces\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "start", "options": [ "end", "evenly", "between", "start", "around", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-002", "family": "distribution", "group_id": "distribution-start-light-1", "image_filename": "distribution-start-light-1.png", "image_sha256": "560524dd93b7f70b5c26f8209854f407b5a654a4ba9891ae330f59b45175786e", "ground_truth": { "choice": "F" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Equal outer spaces, each half the size of an interior gap\nB: All interior gaps and both outer spaces are equal\nC: Packed at the end, touching that inner edge\nD: First and last cards touch opposite inner edges; equal gaps between cards\nE: Packed together in the middle, with much larger equal outer spaces\nF: Packed at the start, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "start", "options": [ "around", "evenly", "end", "between", "center", "start" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-003", "family": "distribution", "group_id": "distribution-start-dark-0", "image_filename": "distribution-start-dark-0.png", "image_sha256": "790c3146f9dc58d06c1996bad5505080ae3a558de99ae89f730a8ea9b630e44f", "ground_truth": { "choice": "B" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: Packed at the start, touching that inner edge\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: Equal outer spaces, each half the size of an interior gap\nE: All interior gaps and both outer spaces are equal\nF: Packed together in the middle, with much larger equal outer spaces\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "start", "options": [ "end", "start", "between", "around", "evenly", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-004", "family": "distribution", "group_id": "distribution-start-dark-1", "image_filename": "distribution-start-dark-1.png", "image_sha256": "d864575a8d583a9d95d519e3bd43dee0aea9682de4157f2e0ee81039d193d5ae", "ground_truth": { "choice": "D" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: All interior gaps and both outer spaces are equal\nB: Packed together in the middle, with much larger equal outer spaces\nC: Packed at the end, touching that inner edge\nD: Packed at the start, touching that inner edge\nE: First and last cards touch opposite inner edges; equal gaps between cards\nF: Equal outer spaces, each half the size of an interior gap\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "start", "options": [ "evenly", "center", "end", "start", "between", "around" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-005", "family": "distribution", "group_id": "distribution-center-light-0", "image_filename": "distribution-center-light-0.png", "image_sha256": "13c3926f98cda2973153eb0ba3bedd05993b22cd5dd6cea1cd3b3c131f6682ad", "ground_truth": { "choice": "E" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Equal outer spaces, each half the size of an interior gap\nB: Packed at the start, touching that inner edge\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: All interior gaps and both outer spaces are equal\nE: Packed together in the middle, with much larger equal outer spaces\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "around", "start", "between", "evenly", "center", "end" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-006", "family": "distribution", "group_id": "distribution-center-light-1", "image_filename": "distribution-center-light-1.png", "image_sha256": "8bd42fb49d0f74f0c78e00cf29761b4a502708d81ed47075c6c3c679ed917065", "ground_truth": { "choice": "E" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: All interior gaps and both outer spaces are equal\nB: Packed at the end, touching that inner edge\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: Packed at the start, touching that inner edge\nE: Packed together in the middle, with much larger equal outer spaces\nF: Equal outer spaces, each half the size of an interior gap\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "evenly", "end", "between", "start", "center", "around" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-007", "family": "distribution", "group_id": "distribution-center-dark-0", "image_filename": "distribution-center-dark-0.png", "image_sha256": "17a87c1e7fa01a4df18a1aff869fafcb6802f07f1e5bcd2a346ed8e438bff341", "ground_truth": { "choice": "B" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Equal outer spaces, each half the size of an interior gap\nB: Packed together in the middle, with much larger equal outer spaces\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: All interior gaps and both outer spaces are equal\nE: Packed at the start, touching that inner edge\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "around", "center", "between", "evenly", "start", "end" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-008", "family": "distribution", "group_id": "distribution-center-dark-1", "image_filename": "distribution-center-dark-1.png", "image_sha256": "52dfc295a559681a6d9dc1869036b888253a906071c83f6bdc5d06a79b0303bb", "ground_truth": { "choice": "D" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: Equal outer spaces, each half the size of an interior gap\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: Packed together in the middle, with much larger equal outer spaces\nE: All interior gaps and both outer spaces are equal\nF: Packed at the start, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "end", "around", "between", "center", "evenly", "start" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-009", "family": "distribution", "group_id": "distribution-end-light-0", "image_filename": "distribution-end-light-0.png", "image_sha256": "0f1f44e01f09213d9b8bece93b73369403c328ce4e1e6940178615f2b86b4377", "ground_truth": { "choice": "A" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: Packed at the start, touching that inner edge\nC: Equal outer spaces, each half the size of an interior gap\nD: All interior gaps and both outer spaces are equal\nE: First and last cards touch opposite inner edges; equal gaps between cards\nF: Packed together in the middle, with much larger equal outer spaces\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "end", "options": [ "end", "start", "around", "evenly", "between", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-010", "family": "distribution", "group_id": "distribution-end-light-1", "image_filename": "distribution-end-light-1.png", "image_sha256": "53bb5e19abdd405e840d6ced432488840f4df2a5f6b756a3b85f373891e1e9eb", "ground_truth": { "choice": "E" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the start, touching that inner edge\nB: First and last cards touch opposite inner edges; equal gaps between cards\nC: All interior gaps and both outer spaces are equal\nD: Packed together in the middle, with much larger equal outer spaces\nE: Packed at the end, touching that inner edge\nF: Equal outer spaces, each half the size of an interior gap\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "end", "options": [ "start", "between", "evenly", "center", "end", "around" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-011", "family": "distribution", "group_id": "distribution-end-dark-0", "image_filename": "distribution-end-dark-0.png", "image_sha256": "34d41ee3ce9e087fe3964190afacf247479019abaf31ebc0951e6ae733b789da", "ground_truth": { "choice": "A" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: Equal outer spaces, each half the size of an interior gap\nC: All interior gaps and both outer spaces are equal\nD: First and last cards touch opposite inner edges; equal gaps between cards\nE: Packed at the start, touching that inner edge\nF: Packed together in the middle, with much larger equal outer spaces\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "end", "options": [ "end", "around", "evenly", "between", "start", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-012", "family": "distribution", "group_id": "distribution-end-dark-1", "image_filename": "distribution-end-dark-1.png", "image_sha256": "aebd5f15e877aa03cdce4a3ec9c330f1fd9d0bb855b03c46294777c28a61a4b0", "ground_truth": { "choice": "F" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the start, touching that inner edge\nB: Packed together in the middle, with much larger equal outer spaces\nC: All interior gaps and both outer spaces are equal\nD: First and last cards touch opposite inner edges; equal gaps between cards\nE: Equal outer spaces, each half the size of an interior gap\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "end", "options": [ "start", "center", "evenly", "between", "around", "end" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-013", "family": "distribution", "group_id": "distribution-between-light-0", "image_filename": "distribution-between-light-0.png", "image_sha256": "a23b333539717be6f0e6ddd82f7b50e5140e9848c24411f1aac5986ba15cb22d", "ground_truth": { "choice": "E" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: Packed together in the middle, with much larger equal outer spaces\nC: Equal outer spaces, each half the size of an interior gap\nD: Packed at the start, touching that inner edge\nE: First and last cards touch opposite inner edges; equal gaps between cards\nF: All interior gaps and both outer spaces are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "between", "options": [ "end", "center", "around", "start", "between", "evenly" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-014", "family": "distribution", "group_id": "distribution-between-light-1", "image_filename": "distribution-between-light-1.png", "image_sha256": "6c346d11652388193a2cf6f477eac372490c7634ebe9cd08e1b1cab5bc7e067b", "ground_truth": { "choice": "A" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: First and last cards touch opposite inner edges; equal gaps between cards\nB: Packed at the start, touching that inner edge\nC: Packed together in the middle, with much larger equal outer spaces\nD: All interior gaps and both outer spaces are equal\nE: Equal outer spaces, each half the size of an interior gap\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "between", "options": [ "between", "start", "center", "evenly", "around", "end" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-015", "family": "distribution", "group_id": "distribution-between-dark-0", "image_filename": "distribution-between-dark-0.png", "image_sha256": "bbb7973d3d6383e82636e2e44b0d85832cce3c07ec05fb8782370843df3a95d1", "ground_truth": { "choice": "C" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: All interior gaps and both outer spaces are equal\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: Equal outer spaces, each half the size of an interior gap\nE: Packed at the start, touching that inner edge\nF: Packed together in the middle, with much larger equal outer spaces\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "between", "options": [ "end", "evenly", "between", "around", "start", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-016", "family": "distribution", "group_id": "distribution-between-dark-1", "image_filename": "distribution-between-dark-1.png", "image_sha256": "2d1a97f606ce03901126522702df733c5d44eba62fd3ad5dce41960ca5b95beb", "ground_truth": { "choice": "C" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Equal outer spaces, each half the size of an interior gap\nB: Packed at the end, touching that inner edge\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: All interior gaps and both outer spaces are equal\nE: Packed together in the middle, with much larger equal outer spaces\nF: Packed at the start, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "between", "options": [ "around", "end", "between", "evenly", "center", "start" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-017", "family": "distribution", "group_id": "distribution-around-light-0", "image_filename": "distribution-around-light-0.png", "image_sha256": "c8df91b1509d235baf54176aecad2860fd8e1f7d3b689e2f51b5a8fa1f0b5358", "ground_truth": { "choice": "F" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the start, touching that inner edge\nB: Packed at the end, touching that inner edge\nC: All interior gaps and both outer spaces are equal\nD: First and last cards touch opposite inner edges; equal gaps between cards\nE: Packed together in the middle, with much larger equal outer spaces\nF: Equal outer spaces, each half the size of an interior gap\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "around", "options": [ "start", "end", "evenly", "between", "center", "around" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-018", "family": "distribution", "group_id": "distribution-around-light-1", "image_filename": "distribution-around-light-1.png", "image_sha256": "929150f26d59093cadf1318a5b1f7adc98d5eb94059522d1260fd864509097e1", "ground_truth": { "choice": "B" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: First and last cards touch opposite inner edges; equal gaps between cards\nB: Equal outer spaces, each half the size of an interior gap\nC: All interior gaps and both outer spaces are equal\nD: Packed together in the middle, with much larger equal outer spaces\nE: Packed at the start, touching that inner edge\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "around", "options": [ "between", "around", "evenly", "center", "start", "end" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-019", "family": "distribution", "group_id": "distribution-around-dark-0", "image_filename": "distribution-around-dark-0.png", "image_sha256": "93a4b61825ca5ee5a9c388742f7bc77ec952dc7aac5da336b7cb812d629bea58", "ground_truth": { "choice": "D" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: First and last cards touch opposite inner edges; equal gaps between cards\nB: Packed at the start, touching that inner edge\nC: Packed at the end, touching that inner edge\nD: Equal outer spaces, each half the size of an interior gap\nE: Packed together in the middle, with much larger equal outer spaces\nF: All interior gaps and both outer spaces are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "around", "options": [ "between", "start", "end", "around", "center", "evenly" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-020", "family": "distribution", "group_id": "distribution-around-dark-1", "image_filename": "distribution-around-dark-1.png", "image_sha256": "c5bee743a5cdf0f55ad8f8e2ad4d0da59843024cf9f0520da9f216dd3e698a27", "ground_truth": { "choice": "F" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: All interior gaps and both outer spaces are equal\nB: Packed together in the middle, with much larger equal outer spaces\nC: Packed at the start, touching that inner edge\nD: Packed at the end, touching that inner edge\nE: First and last cards touch opposite inner edges; equal gaps between cards\nF: Equal outer spaces, each half the size of an interior gap\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "around", "options": [ "evenly", "center", "start", "end", "between", "around" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-021", "family": "distribution", "group_id": "distribution-evenly-light-0", "image_filename": "distribution-evenly-light-0.png", "image_sha256": "4b0fa8ccce73a4842ec16bf47515acc5e5ac08922350410eb12ff459a45e6510", "ground_truth": { "choice": "B" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: First and last cards touch opposite inner edges; equal gaps between cards\nB: All interior gaps and both outer spaces are equal\nC: Packed at the end, touching that inner edge\nD: Equal outer spaces, each half the size of an interior gap\nE: Packed together in the middle, with much larger equal outer spaces\nF: Packed at the start, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "evenly", "options": [ "between", "evenly", "end", "around", "center", "start" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-022", "family": "distribution", "group_id": "distribution-evenly-light-1", "image_filename": "distribution-evenly-light-1.png", "image_sha256": "43750ea9b51206622f9572020ec769262ab52a0a187b81d5f06e5f27b420418a", "ground_truth": { "choice": "C" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: First and last cards touch opposite inner edges; equal gaps between cards\nB: Packed at the start, touching that inner edge\nC: All interior gaps and both outer spaces are equal\nD: Equal outer spaces, each half the size of an interior gap\nE: Packed together in the middle, with much larger equal outer spaces\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "evenly", "options": [ "between", "start", "evenly", "around", "center", "end" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-023", "family": "distribution", "group_id": "distribution-evenly-dark-0", "image_filename": "distribution-evenly-dark-0.png", "image_sha256": "33cc4f41d7465f2ccc190c633d0d7d72468749ededb015c4f8094951e6de1c9f", "ground_truth": { "choice": "A" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: All interior gaps and both outer spaces are equal\nB: Equal outer spaces, each half the size of an interior gap\nC: Packed together in the middle, with much larger equal outer spaces\nD: First and last cards touch opposite inner edges; equal gaps between cards\nE: Packed at the start, touching that inner edge\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "evenly", "options": [ "evenly", "around", "center", "between", "start", "end" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-024", "family": "distribution", "group_id": "distribution-evenly-dark-1", "image_filename": "distribution-evenly-dark-1.png", "image_sha256": "a4233e3b1be0fd9f809700538a6fec7a18fa9a1028c25109dd2a5067d5f71c97", "ground_truth": { "choice": "C" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed together in the middle, with much larger equal outer spaces\nB: First and last cards touch opposite inner edges; equal gaps between cards\nC: All interior gaps and both outer spaces are equal\nD: Packed at the start, touching that inner edge\nE: Equal outer spaces, each half the size of an interior gap\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "evenly", "options": [ "center", "between", "evenly", "start", "around", "end" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-001", "family": "crossalign", "group_id": "crossalign-start-light-0", "image_filename": "crossalign-start-light-0.png", "image_sha256": "6807ee4d1058fe188ee5fa9d536c5fcb6d1cbe8fd637b8d0beab1a91852be6d1", "ground_truth": { "choice": "B" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box fills the frame along that cross axis\nD: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "start", "options": [ "center", "start", "stretch", "end" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-002", "family": "crossalign", "group_id": "crossalign-start-light-1", "image_filename": "crossalign-start-light-1.png", "image_sha256": "0cae12efbadf95299208c915af596834d304f9733727a30f7fbee377b19d5049", "ground_truth": { "choice": "C" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box fills the frame along that cross axis\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "start", "options": [ "center", "end", "start", "stretch" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-003", "family": "crossalign", "group_id": "crossalign-start-dark-0", "image_filename": "crossalign-start-dark-0.png", "image_sha256": "aa0f2e839dd073a842ee9ca332cc700c32e3500e36189064d19376517955a337", "ground_truth": { "choice": "A" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box fills the frame along that cross axis\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "start", "options": [ "start", "stretch", "end", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-004", "family": "crossalign", "group_id": "crossalign-start-dark-1", "image_filename": "crossalign-start-dark-1.png", "image_sha256": "7f4312cb95176955c2fa3e79c7d57089b4cc2a2be6fd78c7257dfb3150b77d72", "ground_truth": { "choice": "D" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box fills the frame along that cross axis\nB: Every box is centered on that cross axis, with empty space on both sides of the box\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "start", "options": [ "stretch", "center", "end", "start" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-005", "family": "crossalign", "group_id": "crossalign-center-light-0", "image_filename": "crossalign-center-light-0.png", "image_sha256": "f13bac549430082588ecdbbff1f7a18b203eff675ffda99f97236b94bcfd583d", "ground_truth": { "choice": "A" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box fills the frame along that cross axis\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "center", "stretch", "end", "start" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-006", "family": "crossalign", "group_id": "crossalign-center-light-1", "image_filename": "crossalign-center-light-1.png", "image_sha256": "5ad39e8f0bb40f9dc4206073ab07b3eb472d8a62234eb00c2d6242b6f8297fe1", "ground_truth": { "choice": "D" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box fills the frame along that cross axis\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "start", "stretch", "end", "center" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-007", "family": "crossalign", "group_id": "crossalign-center-dark-0", "image_filename": "crossalign-center-dark-0.png", "image_sha256": "720dd90c5a75cc47e8d7b4d0707fe3e587c4c3777ebced9d28c59ee8f1f6f976", "ground_truth": { "choice": "D" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box fills the frame along that cross axis\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "start", "stretch", "end", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-008", "family": "crossalign", "group_id": "crossalign-center-dark-1", "image_filename": "crossalign-center-dark-1.png", "image_sha256": "96ba038b9c1e15b95e02d28c9c0a0a669468dbaf0b0e7d9a0c1219e14a988a03", "ground_truth": { "choice": "A" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box fills the frame along that cross axis\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "center", "end", "start", "stretch" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-009", "family": "crossalign", "group_id": "crossalign-end-light-0", "image_filename": "crossalign-end-light-0.png", "image_sha256": "71494b9eb2615d6b9b01b12bc414029d27e898f4b876eacadd29a95689c57800", "ground_truth": { "choice": "A" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box is centered on that cross axis, with empty space on both sides of the box\nC: Every box fills the frame along that cross axis\nD: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "end", "options": [ "end", "center", "stretch", "start" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-010", "family": "crossalign", "group_id": "crossalign-end-light-1", "image_filename": "crossalign-end-light-1.png", "image_sha256": "a3effd6b3a5d358a1b500bd4dbd255c12c9b4ee4e38f59f4f8e7fc21474409bb", "ground_truth": { "choice": "C" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box is centered on that cross axis, with empty space on both sides of the box\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box fills the frame along that cross axis\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "end", "options": [ "start", "center", "end", "stretch" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-011", "family": "crossalign", "group_id": "crossalign-end-dark-0", "image_filename": "crossalign-end-dark-0.png", "image_sha256": "b98a71784ef92ee7def38fc19bdb426fece3ab21027d5470f622150e822a2f62", "ground_truth": { "choice": "C" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box fills the frame along that cross axis\nB: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "end", "options": [ "stretch", "start", "end", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-012", "family": "crossalign", "group_id": "crossalign-end-dark-1", "image_filename": "crossalign-end-dark-1.png", "image_sha256": "0f154b21beb82164b6c260a6854bf151614a4d774e77176c9e52dc93db112b2e", "ground_truth": { "choice": "B" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box fills the frame along that cross axis\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "end", "options": [ "start", "end", "stretch", "center" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-013", "family": "crossalign", "group_id": "crossalign-stretch-light-0", "image_filename": "crossalign-stretch-light-0.png", "image_sha256": "53a6b5623c72652a94e534ed326d0df08ba8974c4ffe15355782b84a94649797", "ground_truth": { "choice": "C" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box is centered on that cross axis, with empty space on both sides of the box\nC: Every box fills the frame along that cross axis\nD: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "stretch", "options": [ "end", "center", "stretch", "start" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-014", "family": "crossalign", "group_id": "crossalign-stretch-light-1", "image_filename": "crossalign-stretch-light-1.png", "image_sha256": "eb41f2095a8ca7c1512aa2814ea9df3855e816dbc2b43d13007c1606a4970ac0", "ground_truth": { "choice": "B" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box fills the frame along that cross axis\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "stretch", "options": [ "center", "stretch", "end", "start" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-015", "family": "crossalign", "group_id": "crossalign-stretch-dark-0", "image_filename": "crossalign-stretch-dark-0.png", "image_sha256": "4c3ef31f845a579e2c208a6d64e9cf419d03b44cd29f4552cdf9b43bfe8c8a34", "ground_truth": { "choice": "D" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box fills the frame along that cross axis\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "stretch", "options": [ "center", "end", "start", "stretch" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-016", "family": "crossalign", "group_id": "crossalign-stretch-dark-1", "image_filename": "crossalign-stretch-dark-1.png", "image_sha256": "f41b2bbfe51e1d891395bf465a844bd5437136fb1ebf12f686255f22018471f7", "ground_truth": { "choice": "B" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box fills the frame along that cross axis\nC: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "stretch", "options": [ "end", "stretch", "start", "center" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-textalign-001", "family": "textalign", "group_id": "textalign-left-light-0", "image_filename": "textalign-left-light-0.png", "image_sha256": "8905dae024a605d690165f00c9acb675ae2c30148a35af4664128c8df89a3a88", "ground_truth": { "choice": "D" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "right", "center", "justify", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 456, "height": 21 }, { "x": 161, "y": 136.890625, "width": 455, "height": 21 }, { "x": 161, "y": 164.78125, "width": 447, "height": 21 }, { "x": 161, "y": 192.671875, "width": 472, "height": 21 }, { "x": 161, "y": 220.5625, "width": 447, "height": 21 }, { "x": 161, "y": 248.453125, "width": 444, "height": 21 }, { "x": 161, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-002", "family": "textalign", "group_id": "textalign-left-light-1", "image_filename": "textalign-left-light-1.png", "image_sha256": "ee1471024f264f11528a7c363c7042a96114b65dafaf828365f21c0ee8b18f61", "ground_truth": { "choice": "D" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "right", "center", "justify", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 477, "height": 21 }, { "x": 161, "y": 164.78125, "width": 447, "height": 21 }, { "x": 161, "y": 192.671875, "width": 468, "height": 21 }, { "x": 161, "y": 220.5625, "width": 473, "height": 21 }, { "x": 161, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-003", "family": "textalign", "group_id": "textalign-left-dark-0", "image_filename": "textalign-left-dark-0.png", "image_sha256": "1348d5f6f99b1b3052c1122cdf87f87f4b89988bb44fe5441d0b56a16e3ebd47", "ground_truth": { "choice": "A" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Left edges line up; right edges are ragged\nB: Right edges line up; left edges are ragged\nC: Line centers line up; both edges are ragged\nD: Both left and right edges line up on the nonfinal lines\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "left", "right", "center", "justify" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 456, "height": 21 }, { "x": 161, "y": 136.890625, "width": 455, "height": 21 }, { "x": 161, "y": 164.78125, "width": 447, "height": 21 }, { "x": 161, "y": 192.671875, "width": 472, "height": 21 }, { "x": 161, "y": 220.5625, "width": 447, "height": 21 }, { "x": 161, "y": 248.453125, "width": 444, "height": 21 }, { "x": 161, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-004", "family": "textalign", "group_id": "textalign-left-dark-1", "image_filename": "textalign-left-dark-1.png", "image_sha256": "690dfd731d6e5ff618ff3c0a1e4cb2d0acd4c750841bd95380926277bc1a70ab", "ground_truth": { "choice": "D" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Both left and right edges line up on the nonfinal lines\nB: Line centers line up; both edges are ragged\nC: Right edges line up; left edges are ragged\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "justify", "center", "right", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 477, "height": 21 }, { "x": 161, "y": 164.78125, "width": 447, "height": 21 }, { "x": 161, "y": 192.671875, "width": 468, "height": 21 }, { "x": 161, "y": 220.5625, "width": 473, "height": 21 }, { "x": 161, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-005", "family": "textalign", "group_id": "textalign-center-light-0", "image_filename": "textalign-center-light-0.png", "image_sha256": "b594092feff844f3f0fcdf6b1e53ada3d71dbd3de071f8e9c2bf69f62a9c44fe", "ground_truth": { "choice": "C" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Left edges line up; right edges are ragged\nB: Right edges line up; left edges are ragged\nC: Line centers line up; both edges are ragged\nD: Both left and right edges line up on the nonfinal lines\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "left", "right", "center", "justify" ], "factors": {}, "lines": { "text-0": [ { "x": 172, "y": 109, "width": 456, "height": 21 }, { "x": 172.5, "y": 136.890625, "width": 455, "height": 21 }, { "x": 176.5, "y": 164.78125, "width": 447, "height": 21 }, { "x": 164, "y": 192.671875, "width": 472, "height": 21 }, { "x": 176.5, "y": 220.5625, "width": 447, "height": 21 }, { "x": 178, "y": 248.453125, "width": 444, "height": 21 }, { "x": 301, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-006", "family": "textalign", "group_id": "textalign-center-light-1", "image_filename": "textalign-center-light-1.png", "image_sha256": "b02b85c997c0d88abe7f20411efc5cb120b5eb8425bac446f1571f488162549b", "ground_truth": { "choice": "B" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Both left and right edges line up on the nonfinal lines\nB: Line centers line up; both edges are ragged\nC: Left edges line up; right edges are ragged\nD: Right edges line up; left edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "justify", "center", "left", "right" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161.5, "y": 136.890625, "width": 477, "height": 21 }, { "x": 176.5, "y": 164.78125, "width": 447, "height": 21 }, { "x": 166, "y": 192.671875, "width": 468, "height": 21 }, { "x": 163.5, "y": 220.5625, "width": 473, "height": 21 }, { "x": 180, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-007", "family": "textalign", "group_id": "textalign-center-dark-0", "image_filename": "textalign-center-dark-0.png", "image_sha256": "872023a65bac1e205fa0a7212f44da2c74dc327d6d1c3bef2433a7372394dbe1", "ground_truth": { "choice": "B" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Left edges line up; right edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Right edges line up; left edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "left", "center", "justify", "right" ], "factors": {}, "lines": { "text-0": [ { "x": 172, "y": 109, "width": 456, "height": 21 }, { "x": 172.5, "y": 136.890625, "width": 455, "height": 21 }, { "x": 176.5, "y": 164.78125, "width": 447, "height": 21 }, { "x": 164, "y": 192.671875, "width": 472, "height": 21 }, { "x": 176.5, "y": 220.5625, "width": 447, "height": 21 }, { "x": 178, "y": 248.453125, "width": 444, "height": 21 }, { "x": 301, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-008", "family": "textalign", "group_id": "textalign-center-dark-1", "image_filename": "textalign-center-dark-1.png", "image_sha256": "45b0cd0ec48f539edae5b70512f48d1c6e420ae9b7645352ba41b60a3e66c3b1", "ground_truth": { "choice": "B" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "right", "center", "justify", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161.5, "y": 136.890625, "width": 477, "height": 21 }, { "x": 176.5, "y": 164.78125, "width": 447, "height": 21 }, { "x": 166, "y": 192.671875, "width": 468, "height": 21 }, { "x": 163.5, "y": 220.5625, "width": 473, "height": 21 }, { "x": 180, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-009", "family": "textalign", "group_id": "textalign-right-light-0", "image_filename": "textalign-right-light-0.png", "image_sha256": "c5d24cf77febfe0fb2e3f5fc92a1bb797941653bb7935afa134bdb8e7c45ce04", "ground_truth": { "choice": "B" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Line centers line up; both edges are ragged\nB: Right edges line up; left edges are ragged\nC: Left edges line up; right edges are ragged\nD: Both left and right edges line up on the nonfinal lines\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "center", "right", "left", "justify" ], "factors": {}, "lines": { "text-0": [ { "x": 183, "y": 109, "width": 456, "height": 21 }, { "x": 184, "y": 136.890625, "width": 455, "height": 21 }, { "x": 192, "y": 164.78125, "width": 447, "height": 21 }, { "x": 167, "y": 192.671875, "width": 472, "height": 21 }, { "x": 192, "y": 220.5625, "width": 447, "height": 21 }, { "x": 195, "y": 248.453125, "width": 444, "height": 21 }, { "x": 441, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-010", "family": "textalign", "group_id": "textalign-right-light-1", "image_filename": "textalign-right-light-1.png", "image_sha256": "878c98bd98d4dde02acdab98e70c7b0400853970164a91c31ec91b364fbace89", "ground_truth": { "choice": "D" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Line centers line up; both edges are ragged\nB: Both left and right edges line up on the nonfinal lines\nC: Left edges line up; right edges are ragged\nD: Right edges line up; left edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "center", "justify", "left", "right" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 162, "y": 136.890625, "width": 477, "height": 21 }, { "x": 192, "y": 164.78125, "width": 447, "height": 21 }, { "x": 171, "y": 192.671875, "width": 468, "height": 21 }, { "x": 166, "y": 220.5625, "width": 473, "height": 21 }, { "x": 199, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-011", "family": "textalign", "group_id": "textalign-right-dark-0", "image_filename": "textalign-right-dark-0.png", "image_sha256": "1ef0c33a06dccfc603e96da71ed17ce5fe3c73cf4515e4c3a04ca269c0745a53", "ground_truth": { "choice": "A" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Both left and right edges line up on the nonfinal lines\nC: Left edges line up; right edges are ragged\nD: Line centers line up; both edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "right", "justify", "left", "center" ], "factors": {}, "lines": { "text-0": [ { "x": 183, "y": 109, "width": 456, "height": 21 }, { "x": 184, "y": 136.890625, "width": 455, "height": 21 }, { "x": 192, "y": 164.78125, "width": 447, "height": 21 }, { "x": 167, "y": 192.671875, "width": 472, "height": 21 }, { "x": 192, "y": 220.5625, "width": 447, "height": 21 }, { "x": 195, "y": 248.453125, "width": 444, "height": 21 }, { "x": 441, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-012", "family": "textalign", "group_id": "textalign-right-dark-1", "image_filename": "textalign-right-dark-1.png", "image_sha256": "75f7c9502d145943c1d28d485e0f5c7242aa7c65081d473a3c69eabd2b4572b0", "ground_truth": { "choice": "A" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Left edges line up; right edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Line centers line up; both edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "right", "left", "justify", "center" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 162, "y": 136.890625, "width": 477, "height": 21 }, { "x": 192, "y": 164.78125, "width": 447, "height": 21 }, { "x": 171, "y": 192.671875, "width": 468, "height": 21 }, { "x": 166, "y": 220.5625, "width": 473, "height": 21 }, { "x": 199, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-013", "family": "textalign", "group_id": "textalign-justify-light-0", "image_filename": "textalign-justify-light-0.png", "image_sha256": "15f73d663b263c3199bc2af63f82b7006773fb0d85d4555b56c25f121110b02e", "ground_truth": { "choice": "A" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Both left and right edges line up on the nonfinal lines\nB: Line centers line up; both edges are ragged\nC: Right edges line up; left edges are ragged\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "justify", "options": [ "justify", "center", "right", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 478, "height": 21 }, { "x": 161, "y": 164.78125, "width": 478, "height": 21 }, { "x": 161, "y": 192.671875, "width": 478, "height": 21 }, { "x": 161, "y": 220.5625, "width": 478, "height": 21 }, { "x": 161, "y": 248.453125, "width": 478, "height": 21 }, { "x": 161, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-014", "family": "textalign", "group_id": "textalign-justify-light-1", "image_filename": "textalign-justify-light-1.png", "image_sha256": "0d6d4e7e771c5395d582ea32b75e5f05ed47b24f35cb10c6df00e9aea3e89c10", "ground_truth": { "choice": "C" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "justify", "options": [ "right", "center", "justify", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 478, "height": 21 }, { "x": 161, "y": 164.78125, "width": 478, "height": 21 }, { "x": 161, "y": 192.671875, "width": 478, "height": 21 }, { "x": 161, "y": 220.5625, "width": 478, "height": 21 }, { "x": 161, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-015", "family": "textalign", "group_id": "textalign-justify-dark-0", "image_filename": "textalign-justify-dark-0.png", "image_sha256": "772229d3293d13622df3a014b260d2337cce0625b141f5aabf5fcdf60c81b542", "ground_truth": { "choice": "C" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Left edges line up; right edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Right edges line up; left edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "justify", "options": [ "left", "center", "justify", "right" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 478, "height": 21 }, { "x": 161, "y": 164.78125, "width": 478, "height": 21 }, { "x": 161, "y": 192.671875, "width": 478, "height": 21 }, { "x": 161, "y": 220.5625, "width": 478, "height": 21 }, { "x": 161, "y": 248.453125, "width": 478, "height": 21 }, { "x": 161, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-016", "family": "textalign", "group_id": "textalign-justify-dark-1", "image_filename": "textalign-justify-dark-1.png", "image_sha256": "0afbc53f0bbe07c398b162d83ab208a158e0bdcfdbcebc35ad1c1277226694a5", "ground_truth": { "choice": "C" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "justify", "options": [ "right", "center", "justify", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 478, "height": 21 }, { "x": 161, "y": 164.78125, "width": 478, "height": 21 }, { "x": 161, "y": 192.671875, "width": 478, "height": 21 }, { "x": 161, "y": 220.5625, "width": 478, "height": 21 }, { "x": 161, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-001", "family": "textcolumns", "group_id": "textcolumns-1-light-0", "image_filename": "textcolumns-1-light-0.png", "image_sha256": "647a54a44dbdf58ed0c52f65c4005d87df8a7a897cfae12ef5a20936a48159d6", "ground_truth": { "choice": "A" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: One\nB: Two\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "1", "options": [ "1", "2", "3" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 582, "height": 21 }, { "x": 96, "y": 136.890625, "width": 587, "height": 21 }, { "x": 96, "y": 164.78125, "width": 603, "height": 21 }, { "x": 96, "y": 192.671875, "width": 564, "height": 21 }, { "x": 96, "y": 220.5625, "width": 595, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-002", "family": "textcolumns", "group_id": "textcolumns-1-light-1", "image_filename": "textcolumns-1-light-1.png", "image_sha256": "40803215467b1223ccfda17fb9ab588c5b4c9adb3540352f6e95fced4c21b653", "ground_truth": { "choice": "B" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: One\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "1", "options": [ "3", "1", "2" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 608, "height": 21 }, { "x": 96, "y": 136.890625, "width": 608, "height": 21 }, { "x": 96, "y": 164.78125, "width": 608, "height": 21 }, { "x": 96, "y": 192.671875, "width": 608, "height": 21 }, { "x": 96, "y": 220.5625, "width": 589, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-003", "family": "textcolumns", "group_id": "textcolumns-1-dark-0", "image_filename": "textcolumns-1-dark-0.png", "image_sha256": "27b5472a838867447f80e48badcf1d9d2e3f0ae65da1397310c70dbfddf56e98", "ground_truth": { "choice": "B" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Two\nB: One\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "1", "options": [ "2", "1", "3" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 582, "height": 21 }, { "x": 96, "y": 136.890625, "width": 587, "height": 21 }, { "x": 96, "y": 164.78125, "width": 603, "height": 21 }, { "x": 96, "y": 192.671875, "width": 564, "height": 21 }, { "x": 96, "y": 220.5625, "width": 595, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-004", "family": "textcolumns", "group_id": "textcolumns-1-dark-1", "image_filename": "textcolumns-1-dark-1.png", "image_sha256": "7df9505249c2768203f9ff31775983aa126e6baa54a7efa86615b24c1f5a68f7", "ground_truth": { "choice": "B" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: One\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "1", "options": [ "3", "1", "2" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 608, "height": 21 }, { "x": 96, "y": 136.890625, "width": 608, "height": 21 }, { "x": 96, "y": 164.78125, "width": 608, "height": 21 }, { "x": 96, "y": 192.671875, "width": 608, "height": 21 }, { "x": 96, "y": 220.5625, "width": 589, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-005", "family": "textcolumns", "group_id": "textcolumns-2-light-0", "image_filename": "textcolumns-2-light-0.png", "image_sha256": "1b118e44838601277d8a40b7fdf57a57d49f6284a106025703dd7a6cd27a7870", "ground_truth": { "choice": "C" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: One\nB: Three\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "2", "options": [ "1", "3", "2" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 264, "height": 21 }, { "x": 96, "y": 136.890625, "width": 259, "height": 21 }, { "x": 96, "y": 164.78125, "width": 248, "height": 21 }, { "x": 96, "y": 192.671875, "width": 272, "height": 21 }, { "x": 96, "y": 220.5625, "width": 257, "height": 21 }, { "x": 96, "y": 248.453125, "width": 264, "height": 21 }, { "x": 96, "y": 276.34375, "width": 248, "height": 21 }, { "x": 96, "y": 304.234375, "width": 269, "height": 21 }, { "x": 96, "y": 332.125, "width": 225, "height": 21 }, { "x": 96, "y": 360.015625, "width": 281, "height": 21 }, { "x": 96, "y": 387.90625, "width": 219, "height": 21 }, { "x": 96, "y": 415.796875, "width": 83, "height": 21 } ], "text-1": [ { "x": 415, "y": 109, "width": 264, "height": 21 }, { "x": 415, "y": 136.890625, "width": 259, "height": 21 }, { "x": 415, "y": 164.78125, "width": 248, "height": 21 }, { "x": 415, "y": 192.671875, "width": 272, "height": 21 }, { "x": 415, "y": 220.5625, "width": 257, "height": 21 }, { "x": 415, "y": 248.453125, "width": 264, "height": 21 }, { "x": 415, "y": 276.34375, "width": 248, "height": 21 }, { "x": 415, "y": 304.234375, "width": 269, "height": 21 }, { "x": 415, "y": 332.125, "width": 225, "height": 21 }, { "x": 415, "y": 360.015625, "width": 281, "height": 21 }, { "x": 415, "y": 387.90625, "width": 219, "height": 21 }, { "x": 415, "y": 415.796875, "width": 83, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-006", "family": "textcolumns", "group_id": "textcolumns-2-light-1", "image_filename": "textcolumns-2-light-1.png", "image_sha256": "525124e99c513c25cd9e95a5a1258ab03a28b2d979e87620882c0dfb69f08dd4", "ground_truth": { "choice": "B" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: Two\nC: One\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "2", "options": [ "3", "2", "1" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 289, "height": 21 }, { "x": 96, "y": 136.890625, "width": 289, "height": 21 }, { "x": 96, "y": 164.78125, "width": 289, "height": 21 }, { "x": 96, "y": 192.671875, "width": 289, "height": 21 }, { "x": 96, "y": 220.5625, "width": 289, "height": 21 }, { "x": 96, "y": 248.453125, "width": 289, "height": 21 }, { "x": 96, "y": 276.34375, "width": 289, "height": 21 }, { "x": 96, "y": 304.234375, "width": 289, "height": 21 }, { "x": 96, "y": 332.125, "width": 289, "height": 21 }, { "x": 96, "y": 360.015625, "width": 289, "height": 21 }, { "x": 96, "y": 387.90625, "width": 41, "height": 21 } ], "text-1": [ { "x": 415, "y": 109, "width": 289, "height": 21 }, { "x": 415, "y": 136.890625, "width": 289, "height": 21 }, { "x": 415, "y": 164.78125, "width": 289, "height": 21 }, { "x": 415, "y": 192.671875, "width": 289, "height": 21 }, { "x": 415, "y": 220.5625, "width": 289, "height": 21 }, { "x": 415, "y": 248.453125, "width": 289, "height": 21 }, { "x": 415, "y": 276.34375, "width": 289, "height": 21 }, { "x": 415, "y": 304.234375, "width": 289, "height": 21 }, { "x": 415, "y": 332.125, "width": 289, "height": 21 }, { "x": 415, "y": 360.015625, "width": 289, "height": 21 }, { "x": 415, "y": 387.90625, "width": 41, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-007", "family": "textcolumns", "group_id": "textcolumns-2-dark-0", "image_filename": "textcolumns-2-dark-0.png", "image_sha256": "12c99f09af63a4826d287da83f0af98df75b5c8a3284edaf7b095659a18fc382", "ground_truth": { "choice": "A" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Two\nB: One\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "2", "options": [ "2", "1", "3" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 264, "height": 21 }, { "x": 96, "y": 136.890625, "width": 259, "height": 21 }, { "x": 96, "y": 164.78125, "width": 248, "height": 21 }, { "x": 96, "y": 192.671875, "width": 272, "height": 21 }, { "x": 96, "y": 220.5625, "width": 257, "height": 21 }, { "x": 96, "y": 248.453125, "width": 264, "height": 21 }, { "x": 96, "y": 276.34375, "width": 248, "height": 21 }, { "x": 96, "y": 304.234375, "width": 269, "height": 21 }, { "x": 96, "y": 332.125, "width": 225, "height": 21 }, { "x": 96, "y": 360.015625, "width": 281, "height": 21 }, { "x": 96, "y": 387.90625, "width": 219, "height": 21 }, { "x": 96, "y": 415.796875, "width": 83, "height": 21 } ], "text-1": [ { "x": 415, "y": 109, "width": 264, "height": 21 }, { "x": 415, "y": 136.890625, "width": 259, "height": 21 }, { "x": 415, "y": 164.78125, "width": 248, "height": 21 }, { "x": 415, "y": 192.671875, "width": 272, "height": 21 }, { "x": 415, "y": 220.5625, "width": 257, "height": 21 }, { "x": 415, "y": 248.453125, "width": 264, "height": 21 }, { "x": 415, "y": 276.34375, "width": 248, "height": 21 }, { "x": 415, "y": 304.234375, "width": 269, "height": 21 }, { "x": 415, "y": 332.125, "width": 225, "height": 21 }, { "x": 415, "y": 360.015625, "width": 281, "height": 21 }, { "x": 415, "y": 387.90625, "width": 219, "height": 21 }, { "x": 415, "y": 415.796875, "width": 83, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-008", "family": "textcolumns", "group_id": "textcolumns-2-dark-1", "image_filename": "textcolumns-2-dark-1.png", "image_sha256": "74ab4de2876fb8d5522ecda1941b9f62d79a0a767b87d50c31b95fb17c71eac0", "ground_truth": { "choice": "C" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: One\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "2", "options": [ "3", "1", "2" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 289, "height": 21 }, { "x": 96, "y": 136.890625, "width": 289, "height": 21 }, { "x": 96, "y": 164.78125, "width": 289, "height": 21 }, { "x": 96, "y": 192.671875, "width": 289, "height": 21 }, { "x": 96, "y": 220.5625, "width": 289, "height": 21 }, { "x": 96, "y": 248.453125, "width": 289, "height": 21 }, { "x": 96, "y": 276.34375, "width": 289, "height": 21 }, { "x": 96, "y": 304.234375, "width": 289, "height": 21 }, { "x": 96, "y": 332.125, "width": 289, "height": 21 }, { "x": 96, "y": 360.015625, "width": 289, "height": 21 }, { "x": 96, "y": 387.90625, "width": 41, "height": 21 } ], "text-1": [ { "x": 415, "y": 109, "width": 289, "height": 21 }, { "x": 415, "y": 136.890625, "width": 289, "height": 21 }, { "x": 415, "y": 164.78125, "width": 289, "height": 21 }, { "x": 415, "y": 192.671875, "width": 289, "height": 21 }, { "x": 415, "y": 220.5625, "width": 289, "height": 21 }, { "x": 415, "y": 248.453125, "width": 289, "height": 21 }, { "x": 415, "y": 276.34375, "width": 289, "height": 21 }, { "x": 415, "y": 304.234375, "width": 289, "height": 21 }, { "x": 415, "y": 332.125, "width": 289, "height": 21 }, { "x": 415, "y": 360.015625, "width": 289, "height": 21 }, { "x": 415, "y": 387.90625, "width": 41, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-009", "family": "textcolumns", "group_id": "textcolumns-3-light-0", "image_filename": "textcolumns-3-light-0.png", "image_sha256": "f2895ce13820d9dd5ed7d55e031fbce6dce0cf407cef425b44a97888998078ae", "ground_truth": { "choice": "A" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: Two\nC: One\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "3", "options": [ "3", "2", "1" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 108, "width": 161, "height": 18 }, { "x": 96, "y": 131.25, "width": 161, "height": 18 }, { "x": 96, "y": 154.5, "width": 168, "height": 18 }, { "x": 96, "y": 177.75, "width": 167, "height": 18 }, { "x": 96, "y": 201, "width": 182, "height": 18 }, { "x": 96, "y": 224.25, "width": 173, "height": 18 }, { "x": 96, "y": 247.5, "width": 147, "height": 18 }, { "x": 96, "y": 270.75, "width": 158, "height": 18 }, { "x": 96, "y": 294, "width": 159, "height": 18 }, { "x": 96, "y": 317.25, "width": 127, "height": 18 }, { "x": 96, "y": 340.5, "width": 150, "height": 18 }, { "x": 96, "y": 363.75, "width": 146, "height": 18 }, { "x": 96, "y": 387, "width": 158, "height": 18 }, { "x": 96, "y": 410.25, "width": 167, "height": 18 }, { "x": 96, "y": 433.5, "width": 142, "height": 18 }, { "x": 96, "y": 456.75, "width": 71, "height": 18 } ], "text-1": [ { "x": 308.65625, "y": 108, "width": 161, "height": 18 }, { "x": 308.65625, "y": 131.25, "width": 161, "height": 18 }, { "x": 308.65625, "y": 154.5, "width": 168, "height": 18 }, { "x": 308.65625, "y": 177.75, "width": 167, "height": 18 }, { "x": 308.65625, "y": 201, "width": 182, "height": 18 }, { "x": 308.65625, "y": 224.25, "width": 173, "height": 18 }, { "x": 308.65625, "y": 247.5, "width": 147, "height": 18 }, { "x": 308.65625, "y": 270.75, "width": 158, "height": 18 }, { "x": 308.65625, "y": 294, "width": 159, "height": 18 }, { "x": 308.65625, "y": 317.25, "width": 127, "height": 18 }, { "x": 308.65625, "y": 340.5, "width": 150, "height": 18 }, { "x": 308.65625, "y": 363.75, "width": 146, "height": 18 }, { "x": 308.65625, "y": 387, "width": 158, "height": 18 }, { "x": 308.65625, "y": 410.25, "width": 167, "height": 18 }, { "x": 308.65625, "y": 433.5, "width": 142, "height": 18 }, { "x": 308.65625, "y": 456.75, "width": 71, "height": 18 } ], "text-2": [ { "x": 521.328125, "y": 108, "width": 161, "height": 18 }, { "x": 521.328125, "y": 131.25, "width": 161, "height": 18 }, { "x": 521.328125, "y": 154.5, "width": 168, "height": 18 }, { "x": 521.328125, "y": 177.75, "width": 167, "height": 18 }, { "x": 521.328125, "y": 201, "width": 182, "height": 18 }, { "x": 521.328125, "y": 224.25, "width": 173, "height": 18 }, { "x": 521.328125, "y": 247.5, "width": 147, "height": 18 }, { "x": 521.328125, "y": 270.75, "width": 158, "height": 18 }, { "x": 521.328125, "y": 294, "width": 159, "height": 18 }, { "x": 521.328125, "y": 317.25, "width": 127, "height": 18 }, { "x": 521.328125, "y": 340.5, "width": 150, "height": 18 }, { "x": 521.328125, "y": 363.75, "width": 146, "height": 18 }, { "x": 521.328125, "y": 387, "width": 158, "height": 18 }, { "x": 521.328125, "y": 410.25, "width": 167, "height": 18 }, { "x": 521.328125, "y": 433.5, "width": 142, "height": 18 }, { "x": 521.328125, "y": 456.75, "width": 71, "height": 18 } ] } } }, { "task_id": "layoutbench-textcolumns-010", "family": "textcolumns", "group_id": "textcolumns-3-light-1", "image_filename": "textcolumns-3-light-1.png", "image_sha256": "be4ccaa0a2d8241899bed3bd19071abb8f6906c288c7b4a99f38abebf33fdef4", "ground_truth": { "choice": "A" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: Two\nC: One\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "3", "options": [ "3", "2", "1" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 108, "width": 182.65625, "height": 18 }, { "x": 96, "y": 131.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 154.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 177.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 201, "width": 182.65625, "height": 18 }, { "x": 96, "y": 224.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 247.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 270.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 294, "width": 182.65625, "height": 18 }, { "x": 96, "y": 317.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 340.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 363.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 387, "width": 182.65625, "height": 18 }, { "x": 96, "y": 410.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 433.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 456.75, "width": 93, "height": 18 } ], "text-1": [ { "x": 308.65625, "y": 108, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 131.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 154.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 177.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 201, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 224.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 247.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 270.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 294, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 317.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 340.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 363.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 387, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 410.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 433.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 456.75, "width": 93, "height": 18 } ], "text-2": [ { "x": 521.328125, "y": 108, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 131.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 154.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 177.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 201, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 224.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 247.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 270.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 294, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 317.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 340.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 363.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 387, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 410.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 433.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 456.75, "width": 93, "height": 18 } ] } } }, { "task_id": "layoutbench-textcolumns-011", "family": "textcolumns", "group_id": "textcolumns-3-dark-0", "image_filename": "textcolumns-3-dark-0.png", "image_sha256": "c6c81995a5ca8f20eae37ab32c85ec41daffada38a97d893c356e682a2a13502", "ground_truth": { "choice": "C" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: One\nB: Two\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "3", "options": [ "1", "2", "3" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 108, "width": 161, "height": 18 }, { "x": 96, "y": 131.25, "width": 161, "height": 18 }, { "x": 96, "y": 154.5, "width": 168, "height": 18 }, { "x": 96, "y": 177.75, "width": 167, "height": 18 }, { "x": 96, "y": 201, "width": 182, "height": 18 }, { "x": 96, "y": 224.25, "width": 173, "height": 18 }, { "x": 96, "y": 247.5, "width": 147, "height": 18 }, { "x": 96, "y": 270.75, "width": 158, "height": 18 }, { "x": 96, "y": 294, "width": 159, "height": 18 }, { "x": 96, "y": 317.25, "width": 127, "height": 18 }, { "x": 96, "y": 340.5, "width": 150, "height": 18 }, { "x": 96, "y": 363.75, "width": 146, "height": 18 }, { "x": 96, "y": 387, "width": 158, "height": 18 }, { "x": 96, "y": 410.25, "width": 167, "height": 18 }, { "x": 96, "y": 433.5, "width": 142, "height": 18 }, { "x": 96, "y": 456.75, "width": 71, "height": 18 } ], "text-1": [ { "x": 308.65625, "y": 108, "width": 161, "height": 18 }, { "x": 308.65625, "y": 131.25, "width": 161, "height": 18 }, { "x": 308.65625, "y": 154.5, "width": 168, "height": 18 }, { "x": 308.65625, "y": 177.75, "width": 167, "height": 18 }, { "x": 308.65625, "y": 201, "width": 182, "height": 18 }, { "x": 308.65625, "y": 224.25, "width": 173, "height": 18 }, { "x": 308.65625, "y": 247.5, "width": 147, "height": 18 }, { "x": 308.65625, "y": 270.75, "width": 158, "height": 18 }, { "x": 308.65625, "y": 294, "width": 159, "height": 18 }, { "x": 308.65625, "y": 317.25, "width": 127, "height": 18 }, { "x": 308.65625, "y": 340.5, "width": 150, "height": 18 }, { "x": 308.65625, "y": 363.75, "width": 146, "height": 18 }, { "x": 308.65625, "y": 387, "width": 158, "height": 18 }, { "x": 308.65625, "y": 410.25, "width": 167, "height": 18 }, { "x": 308.65625, "y": 433.5, "width": 142, "height": 18 }, { "x": 308.65625, "y": 456.75, "width": 71, "height": 18 } ], "text-2": [ { "x": 521.328125, "y": 108, "width": 161, "height": 18 }, { "x": 521.328125, "y": 131.25, "width": 161, "height": 18 }, { "x": 521.328125, "y": 154.5, "width": 168, "height": 18 }, { "x": 521.328125, "y": 177.75, "width": 167, "height": 18 }, { "x": 521.328125, "y": 201, "width": 182, "height": 18 }, { "x": 521.328125, "y": 224.25, "width": 173, "height": 18 }, { "x": 521.328125, "y": 247.5, "width": 147, "height": 18 }, { "x": 521.328125, "y": 270.75, "width": 158, "height": 18 }, { "x": 521.328125, "y": 294, "width": 159, "height": 18 }, { "x": 521.328125, "y": 317.25, "width": 127, "height": 18 }, { "x": 521.328125, "y": 340.5, "width": 150, "height": 18 }, { "x": 521.328125, "y": 363.75, "width": 146, "height": 18 }, { "x": 521.328125, "y": 387, "width": 158, "height": 18 }, { "x": 521.328125, "y": 410.25, "width": 167, "height": 18 }, { "x": 521.328125, "y": 433.5, "width": 142, "height": 18 }, { "x": 521.328125, "y": 456.75, "width": 71, "height": 18 } ] } } }, { "task_id": "layoutbench-textcolumns-012", "family": "textcolumns", "group_id": "textcolumns-3-dark-1", "image_filename": "textcolumns-3-dark-1.png", "image_sha256": "7a2112804a99ff45987bc12cc3b11a9f8ae59924997cf456e5f6904c8e636228", "ground_truth": { "choice": "C" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Two\nB: One\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "3", "options": [ "2", "1", "3" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 108, "width": 182.65625, "height": 18 }, { "x": 96, "y": 131.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 154.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 177.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 201, "width": 182.65625, "height": 18 }, { "x": 96, "y": 224.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 247.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 270.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 294, "width": 182.65625, "height": 18 }, { "x": 96, "y": 317.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 340.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 363.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 387, "width": 182.65625, "height": 18 }, { "x": 96, "y": 410.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 433.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 456.75, "width": 93, "height": 18 } ], "text-1": [ { "x": 308.65625, "y": 108, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 131.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 154.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 177.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 201, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 224.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 247.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 270.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 294, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 317.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 340.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 363.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 387, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 410.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 433.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 456.75, "width": 93, "height": 18 } ], "text-2": [ { "x": 521.328125, "y": 108, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 131.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 154.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 177.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 201, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 224.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 247.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 270.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 294, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 317.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 340.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 363.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 387, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 410.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 433.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 456.75, "width": 93, "height": 18 } ] } } }, { "task_id": "layoutbench-gridcols-001", "family": "gridcols", "group_id": "gridcols-2-light-0", "image_filename": "gridcols-2-light-0.png", "image_sha256": "dd1adba5db8f486b008cacd9f7a2028a6507ad49ac0dca1b4949847f0766c8c5", "ground_truth": { "choice": "A" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "2", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-002", "family": "gridcols", "group_id": "gridcols-2-light-1", "image_filename": "gridcols-2-light-1.png", "image_sha256": "18acda7f7c618a05f19af3a8dbb4fb07588a9ae063feef024e7382c07aea9b8e", "ground_truth": { "choice": "B" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Three\nB: Two\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "2", "options": [ "3", "2", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-003", "family": "gridcols", "group_id": "gridcols-2-dark-0", "image_filename": "gridcols-2-dark-0.png", "image_sha256": "6fa5f99244d01e00f27463ac84f40c12de35ba8eaf860bae7d733ab364e2719d", "ground_truth": { "choice": "B" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Four\nB: Two\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "2", "options": [ "4", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-004", "family": "gridcols", "group_id": "gridcols-2-dark-1", "image_filename": "gridcols-2-dark-1.png", "image_sha256": "2d88c8d5030683a96dee466b06a1b7ea7c529c97f2cd5fcdb52d6c1e5e9510ac", "ground_truth": { "choice": "C" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Four\nB: Three\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "2", "options": [ "4", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-005", "family": "gridcols", "group_id": "gridcols-3-light-0", "image_filename": "gridcols-3-light-0.png", "image_sha256": "812b272d8b53cebe7c9205b8ba1e2e3954c08a0c9c5046e8b6ae392c120da238", "ground_truth": { "choice": "A" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Three\nB: Four\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "3", "options": [ "3", "4", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-006", "family": "gridcols", "group_id": "gridcols-3-light-1", "image_filename": "gridcols-3-light-1.png", "image_sha256": "69534ac5ed527b2dde3d2db25199c2447e246348ba45210cc6b13b58820a515f", "ground_truth": { "choice": "B" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "3", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-007", "family": "gridcols", "group_id": "gridcols-3-dark-0", "image_filename": "gridcols-3-dark-0.png", "image_sha256": "0351f12f5f4f2d3aaf8680babc2d3df35120e94659f2c12ef163521e381c0bb1", "ground_truth": { "choice": "C" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Four\nB: Two\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "3", "options": [ "4", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-008", "family": "gridcols", "group_id": "gridcols-3-dark-1", "image_filename": "gridcols-3-dark-1.png", "image_sha256": "f36d992202281f254b489f04029ec746b38366eab24c49e9acb5351a92d12070", "ground_truth": { "choice": "A" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Three\nB: Four\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "3", "options": [ "3", "4", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-009", "family": "gridcols", "group_id": "gridcols-4-light-0", "image_filename": "gridcols-4-light-0.png", "image_sha256": "e211130610b8d41a59b8cd8fd303fa7db8a2f567813c3147048f9e2da39ef8e9", "ground_truth": { "choice": "C" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "4", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-010", "family": "gridcols", "group_id": "gridcols-4-light-1", "image_filename": "gridcols-4-light-1.png", "image_sha256": "bc6a8cdd59ff4756891a2ce2ca122f9d163a0309f35ce4cfa2be501275912c79", "ground_truth": { "choice": "A" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Four\nB: Three\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "4", "options": [ "4", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-011", "family": "gridcols", "group_id": "gridcols-4-dark-0", "image_filename": "gridcols-4-dark-0.png", "image_sha256": "e39694825eef9f0fb1cfc923574d8533e29bb331a8dcc3242912551a3541e0b4", "ground_truth": { "choice": "B" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Two\nB: Four\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "4", "options": [ "2", "4", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-012", "family": "gridcols", "group_id": "gridcols-4-dark-1", "image_filename": "gridcols-4-dark-1.png", "image_sha256": "9e1de6feb34b56bcf4a18793fc16d571ba0c5e1235f5e1a632dc4d02b2467509", "ground_truth": { "choice": "C" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "4", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-001", "family": "gridrows", "group_id": "gridcols-2-light-0", "image_filename": "gridcols-2-light-0.png", "image_sha256": "dd1adba5db8f486b008cacd9f7a2028a6507ad49ac0dca1b4949847f0766c8c5", "ground_truth": { "choice": "A" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "2", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-002", "family": "gridrows", "group_id": "gridrows-2-light-1", "image_filename": "gridrows-2-light-1.png", "image_sha256": "b009117053a4957a26a13c5912b060e79a8905602543a636e5d5bc24f0bb5fdc", "ground_truth": { "choice": "B" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Three\nB: Two\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "2", "options": [ "3", "2", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-003", "family": "gridrows", "group_id": "gridcols-2-dark-0", "image_filename": "gridcols-2-dark-0.png", "image_sha256": "6fa5f99244d01e00f27463ac84f40c12de35ba8eaf860bae7d733ab364e2719d", "ground_truth": { "choice": "A" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "2", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-004", "family": "gridrows", "group_id": "gridrows-2-dark-1", "image_filename": "gridrows-2-dark-1.png", "image_sha256": "5e9c134086843fcd5447a5084b3ac4b874bc3adf55c621600fecc742d7848b69", "ground_truth": { "choice": "C" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Four\nB: Three\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "2", "options": [ "4", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-005", "family": "gridrows", "group_id": "gridrows-3-light-0", "image_filename": "gridrows-3-light-0.png", "image_sha256": "fc323619c2a79c900e3e6636a884ea6e63b4d11008b77983221ccddf8630431c", "ground_truth": { "choice": "C" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Four\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "3", "options": [ "2", "4", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-006", "family": "gridrows", "group_id": "gridcols-3-light-1", "image_filename": "gridcols-3-light-1.png", "image_sha256": "69534ac5ed527b2dde3d2db25199c2447e246348ba45210cc6b13b58820a515f", "ground_truth": { "choice": "C" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Four\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "3", "options": [ "2", "4", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-007", "family": "gridrows", "group_id": "gridrows-3-dark-0", "image_filename": "gridrows-3-dark-0.png", "image_sha256": "08e8f1cfe3c14b1b92802273a6cf783b7ea92e9ef4e18d20ae389945677f6ce1", "ground_truth": { "choice": "B" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "3", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-008", "family": "gridrows", "group_id": "gridcols-3-dark-1", "image_filename": "gridcols-3-dark-1.png", "image_sha256": "f36d992202281f254b489f04029ec746b38366eab24c49e9acb5351a92d12070", "ground_truth": { "choice": "C" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Four\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "3", "options": [ "2", "4", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-009", "family": "gridrows", "group_id": "gridrows-4-light-0", "image_filename": "gridrows-4-light-0.png", "image_sha256": "0263bdb2d490f291a2ce043686bb6b3dfb903c51bdbbd4d0a530f2221d5ea841", "ground_truth": { "choice": "B" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Four\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "4", "options": [ "2", "4", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-010", "family": "gridrows", "group_id": "gridrows-4-light-1", "image_filename": "gridrows-4-light-1.png", "image_sha256": "1288f934549ad03f39e810de459e88579b0e56973d8d16943ed545e696ce38e5", "ground_truth": { "choice": "B" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Three\nB: Four\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "4", "options": [ "3", "4", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-011", "family": "gridrows", "group_id": "gridrows-4-dark-0", "image_filename": "gridrows-4-dark-0.png", "image_sha256": "a128b8a14494801896465b87c2bb8e40d905e235ae1f582d44a9d47eca5ee818", "ground_truth": { "choice": "A" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Four\nB: Two\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "4", "options": [ "4", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-012", "family": "gridrows", "group_id": "gridrows-4-dark-1", "image_filename": "gridrows-4-dark-1.png", "image_sha256": "3957f045bd97a3caa2d62cc2142d59dc4dc84baac1891cbb3ac37b106d945a4a", "ground_truth": { "choice": "A" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Four\nB: Three\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "4", "options": [ "4", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-001", "family": "gridgaps", "group_id": "gridgaps-horizontal-light-0", "image_filename": "gridgaps-horizontal-light-0.png", "image_sha256": "d1df4b92e915ddce23934a0ee957fdeeca596611f34b69d00d2c977c04a6e51a", "ground_truth": { "choice": "C" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Vertical gutters are larger\nC: Horizontal gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "horizontal", "options": [ "equal", "vertical", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-002", "family": "gridgaps", "group_id": "gridgaps-horizontal-light-1", "image_filename": "gridgaps-horizontal-light-1.png", "image_sha256": "a2a50b94b7d51187e4f2fdb7eb64a670ab44c5ca9ab5e5ee13c7d4d356f93b02", "ground_truth": { "choice": "B" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Horizontal gutters are larger\nC: Vertical gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "horizontal", "options": [ "equal", "horizontal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-003", "family": "gridgaps", "group_id": "gridgaps-horizontal-dark-0", "image_filename": "gridgaps-horizontal-dark-0.png", "image_sha256": "a91d4a48ec49d3ca21202f9c33652a1599677a01df08b79eea7727304de7f146", "ground_truth": { "choice": "B" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Horizontal gutters are larger\nC: Vertical gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "horizontal", "options": [ "equal", "horizontal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-004", "family": "gridgaps", "group_id": "gridgaps-horizontal-dark-1", "image_filename": "gridgaps-horizontal-dark-1.png", "image_sha256": "90160df0c6846be09f70273be7958da4107c42e26621506e429530e0f98d2f0c", "ground_truth": { "choice": "C" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Vertical gutters are larger\nB: The two gutter sizes are equal\nC: Horizontal gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "horizontal", "options": [ "vertical", "equal", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-005", "family": "gridgaps", "group_id": "gridgaps-vertical-light-0", "image_filename": "gridgaps-vertical-light-0.png", "image_sha256": "feb0fd98985c729d1671414e8e0efcd3572776d57c692076cbe3175aadd95f68", "ground_truth": { "choice": "A" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Vertical gutters are larger\nB: Horizontal gutters are larger\nC: The two gutter sizes are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "vertical", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-006", "family": "gridgaps", "group_id": "gridgaps-vertical-light-1", "image_filename": "gridgaps-vertical-light-1.png", "image_sha256": "ec764924ec14bc8efee7302302fd6c57178068714d3c76dc20d67a7a01493948", "ground_truth": { "choice": "C" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Horizontal gutters are larger\nB: The two gutter sizes are equal\nC: Vertical gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "vertical", "options": [ "horizontal", "equal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-007", "family": "gridgaps", "group_id": "gridgaps-vertical-dark-0", "image_filename": "gridgaps-vertical-dark-0.png", "image_sha256": "8573792e663152c325cb5e7b97290691a02f38c1e4a1c02406f12f1bd4b26a9b", "ground_truth": { "choice": "B" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Vertical gutters are larger\nC: Horizontal gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "vertical", "options": [ "equal", "vertical", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-008", "family": "gridgaps", "group_id": "gridgaps-vertical-dark-1", "image_filename": "gridgaps-vertical-dark-1.png", "image_sha256": "751cafd93f1cbd9a21e83671b34e152b80fad69dcf1b4ccbc888340cf467761f", "ground_truth": { "choice": "A" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Vertical gutters are larger\nB: Horizontal gutters are larger\nC: The two gutter sizes are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "vertical", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-009", "family": "gridgaps", "group_id": "gridgaps-equal-light-0", "image_filename": "gridgaps-equal-light-0.png", "image_sha256": "1da20fc6045872218e5145c61d94b6a600a27b80e86335a110050b9fed8704dd", "ground_truth": { "choice": "A" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Vertical gutters are larger\nC: Horizontal gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "equal", "vertical", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-010", "family": "gridgaps", "group_id": "gridcols-3-light-1", "image_filename": "gridcols-3-light-1.png", "image_sha256": "69534ac5ed527b2dde3d2db25199c2447e246348ba45210cc6b13b58820a515f", "ground_truth": { "choice": "A" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Horizontal gutters are larger\nC: Vertical gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "equal", "horizontal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-011", "family": "gridgaps", "group_id": "gridgaps-equal-dark-0", "image_filename": "gridgaps-equal-dark-0.png", "image_sha256": "7665a3776248fd9fce596c339b45ce73d5c2580c85fa6e2991252f0e49c78bda", "ground_truth": { "choice": "B" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Horizontal gutters are larger\nB: The two gutter sizes are equal\nC: Vertical gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "horizontal", "equal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-012", "family": "gridgaps", "group_id": "gridcols-3-dark-1", "image_filename": "gridcols-3-dark-1.png", "image_sha256": "f36d992202281f254b489f04029ec746b38366eab24c49e9acb5351a92d12070", "ground_truth": { "choice": "C" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Horizontal gutters are larger\nB: Vertical gutters are larger\nC: The two gutter sizes are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "horizontal", "vertical", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-001", "family": "gridtracks", "group_id": "gridgaps-equal-light-0", "image_filename": "gridgaps-equal-light-0.png", "image_sha256": "1da20fc6045872218e5145c61d94b6a600a27b80e86335a110050b9fed8704dd", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: The right column is wider; the other two have equal widths\nC: All three columns have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "left", "right", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-002", "family": "gridtracks", "group_id": "gridcols-3-light-1", "image_filename": "gridcols-3-light-1.png", "image_sha256": "69534ac5ed527b2dde3d2db25199c2447e246348ba45210cc6b13b58820a515f", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: All three columns have equal widths\nC: The right column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "left", "equal", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-003", "family": "gridtracks", "group_id": "gridgaps-equal-dark-0", "image_filename": "gridgaps-equal-dark-0.png", "image_sha256": "7665a3776248fd9fce596c339b45ce73d5c2580c85fa6e2991252f0e49c78bda", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The right column is wider; the other two have equal widths\nB: The left column is wider; the other two have equal widths\nC: All three columns have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "right", "left", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-004", "family": "gridtracks", "group_id": "gridcols-3-dark-1", "image_filename": "gridcols-3-dark-1.png", "image_sha256": "f36d992202281f254b489f04029ec746b38366eab24c49e9acb5351a92d12070", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The right column is wider; the other two have equal widths\nB: All three columns have equal widths\nC: The left column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "right", "equal", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-005", "family": "gridtracks", "group_id": "gridtracks-left-light-0", "image_filename": "gridtracks-left-light-0.png", "image_sha256": "f0e212b23db4ed3fd338d44001acedf73a067b4acd417f0be84571c5b1c1194b", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: All three columns have equal widths\nB: The left column is wider; the other two have equal widths\nC: The right column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "equal", "left", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-006", "family": "gridtracks", "group_id": "gridtracks-left-light-1", "image_filename": "gridtracks-left-light-1.png", "image_sha256": "dc9b6fcb5eadf9ee9062cbe849f88fece0711aaedfc79e7d23091c2b14b3453a", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: The right column is wider; the other two have equal widths\nC: All three columns have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "left", "right", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-007", "family": "gridtracks", "group_id": "gridtracks-left-dark-0", "image_filename": "gridtracks-left-dark-0.png", "image_sha256": "4a60bd7e28b6abf63d6da7792b1389ed101663e36c934e18d7e7442b1068b675", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: The right column is wider; the other two have equal widths\nC: All three columns have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "left", "right", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-008", "family": "gridtracks", "group_id": "gridtracks-left-dark-1", "image_filename": "gridtracks-left-dark-1.png", "image_sha256": "cbe9f6b29fe105ea4b597e4cf9b5b060144d6b288c4a1419275bf2dae1c88060", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: All three columns have equal widths\nC: The right column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "left", "equal", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-009", "family": "gridtracks", "group_id": "gridtracks-right-light-0", "image_filename": "gridtracks-right-light-0.png", "image_sha256": "83daf8ff9fcb22586aaf6a8e6f08685f9482c2b784497be7152330d19948f140", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: All three columns have equal widths\nB: The left column is wider; the other two have equal widths\nC: The right column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "equal", "left", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-010", "family": "gridtracks", "group_id": "gridtracks-right-light-1", "image_filename": "gridtracks-right-light-1.png", "image_sha256": "a51b46ab70129e8bb6e796e2ea7f918179eecc486a81197479d87c92e80714ab", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: All three columns have equal widths\nB: The right column is wider; the other two have equal widths\nC: The left column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "equal", "right", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-011", "family": "gridtracks", "group_id": "gridtracks-right-dark-0", "image_filename": "gridtracks-right-dark-0.png", "image_sha256": "2b905573edc99531abce8079598897def2803d677b689e827289ce6ddcffa185", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The right column is wider; the other two have equal widths\nB: All three columns have equal widths\nC: The left column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "right", "equal", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-012", "family": "gridtracks", "group_id": "gridtracks-right-dark-1", "image_filename": "gridtracks-right-dark-1.png", "image_sha256": "978062e15a160e803e73e5afec4bc42eb18130f177742f3dcf740d8d973863dc", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: All three columns have equal widths\nC: The right column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "left", "equal", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-001", "family": "gridspan", "group_id": "gridspan-1-light-0", "image_filename": "gridspan-1-light-0.png", "image_sha256": "5113879de5447472367eb5324a0417433871bd3529aa4fd1b0fca3e1fb588f04", "ground_truth": { "choice": "B" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Three column positions\nB: One column position\nC: Two column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "1", "options": [ "3", "1", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-002", "family": "gridspan", "group_id": "gridspan-1-light-1", "image_filename": "gridspan-1-light-1.png", "image_sha256": "09dad52c37707c839600f15ea6c03ae0e92ed6343e095a863164b615e50dd864", "ground_truth": { "choice": "A" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: One column position\nB: Three column positions\nC: Two column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "1", "options": [ "1", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-003", "family": "gridspan", "group_id": "gridspan-1-dark-0", "image_filename": "gridspan-1-dark-0.png", "image_sha256": "78154cae227f20d2389899d9f67326fee3b1b4da7118129cc02f5048a9f73f91", "ground_truth": { "choice": "C" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "1", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-004", "family": "gridspan", "group_id": "gridspan-1-dark-1", "image_filename": "gridspan-1-dark-1.png", "image_sha256": "a4a1298de87a7d5fa55542610f90f34f5acb449bfdcf54e32103f6ad43bcfcfc", "ground_truth": { "choice": "C" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "1", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-005", "family": "gridspan", "group_id": "gridspan-2-light-0", "image_filename": "gridspan-2-light-0.png", "image_sha256": "278545a076abcbca9cc809b4abeba5cd6ad47b3d10207960ee7b5a8dcb0eaadb", "ground_truth": { "choice": "C" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Three column positions\nB: One column position\nC: Two column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "2", "options": [ "3", "1", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-006", "family": "gridspan", "group_id": "gridspan-2-light-1", "image_filename": "gridspan-2-light-1.png", "image_sha256": "b4896fa3bb1c23f884710e9cf0009678819726c2336bbaf5a9ef90e07b516f79", "ground_truth": { "choice": "A" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "2", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-007", "family": "gridspan", "group_id": "gridspan-2-dark-0", "image_filename": "gridspan-2-dark-0.png", "image_sha256": "c84bc711ebd52999d21ed6d805b2b96728de31f1fa3800d43d933c5732653776", "ground_truth": { "choice": "A" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "2", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-008", "family": "gridspan", "group_id": "gridspan-2-dark-1", "image_filename": "gridspan-2-dark-1.png", "image_sha256": "b717f60b0c9fb8b369bb5c9ee941300b7e69a0da0f76dc59ced325d02b0f1d12", "ground_truth": { "choice": "A" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: One column position\nC: Three column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "2", "options": [ "2", "1", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-009", "family": "gridspan", "group_id": "gridspan-3-light-0", "image_filename": "gridspan-3-light-0.png", "image_sha256": "27cbb3bea8a70078841833f43663b15abc63ce04cb2fdd13f0554fcca8403181", "ground_truth": { "choice": "B" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: One column position\nB: Three column positions\nC: Two column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "3", "options": [ "1", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-010", "family": "gridspan", "group_id": "gridspan-3-light-1", "image_filename": "gridspan-3-light-1.png", "image_sha256": "a08da053dedba4b439a7657a7b565da080619f7c2d798e06bf83b0332662d684", "ground_truth": { "choice": "B" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "3", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-011", "family": "gridspan", "group_id": "gridspan-3-dark-0", "image_filename": "gridspan-3-dark-0.png", "image_sha256": "753e859b97c8e2317e7adaec19cc8e3217a106f93542354c9142624c065d4b25", "ground_truth": { "choice": "B" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "3", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-012", "family": "gridspan", "group_id": "gridspan-3-dark-1", "image_filename": "gridspan-3-dark-1.png", "image_sha256": "6e476c5eb1b07e94a4090068ec8282b846207d210e01096215498232e50fc0a3", "ground_truth": { "choice": "C" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: One column position\nB: Two column positions\nC: Three column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "3", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-001", "family": "wrapcount", "group_id": "wrapcount-1-light-0", "image_filename": "wrapcount-1-light-0.png", "image_sha256": "8068924055f3c867ac47f96855b526f9f0d9e55ab8fbaac9b7ed7e585fb689b5", "ground_truth": { "choice": "A" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Three rows\nC: Two rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "1", "options": [ "1", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-002", "family": "wrapcount", "group_id": "wrapcount-1-light-1", "image_filename": "wrapcount-1-light-1.png", "image_sha256": "b138b6a69d71fb2b1d7a71ca58ae615ee35f2e6fecfd457ad7baf5e3107a2e1f", "ground_truth": { "choice": "A" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Two rows\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "1", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-003", "family": "wrapcount", "group_id": "wrapcount-1-dark-0", "image_filename": "wrapcount-1-dark-0.png", "image_sha256": "4e90c93c89ba8154640bc88379ec46db9a2587de7204e07599befad4162dbaba", "ground_truth": { "choice": "A" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Two rows\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "1", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-004", "family": "wrapcount", "group_id": "wrapcount-1-dark-1", "image_filename": "wrapcount-1-dark-1.png", "image_sha256": "c44a030d6f639864188327ac3752600da70dbf170eabe627c3a4680bde2021a6", "ground_truth": { "choice": "B" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: Two rows\nB: One row\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "1", "options": [ "2", "1", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-005", "family": "wrapcount", "group_id": "wrapcount-2-light-0", "image_filename": "wrapcount-2-light-0.png", "image_sha256": "5f6dd957b89f6660e886cf03eb4bb0b81e6622b22c6bd9a77d7f46901f842a60", "ground_truth": { "choice": "C" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: Three rows\nB: One row\nC: Two rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "2", "options": [ "3", "1", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-006", "family": "wrapcount", "group_id": "wrapcount-2-light-1", "image_filename": "wrapcount-2-light-1.png", "image_sha256": "14588c5f12de52cd51d23e17743279f5a2468f2327046fa33ce54c3a2a42e858", "ground_truth": { "choice": "C" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: Three rows\nB: One row\nC: Two rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "2", "options": [ "3", "1", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-007", "family": "wrapcount", "group_id": "wrapcount-2-dark-0", "image_filename": "wrapcount-2-dark-0.png", "image_sha256": "0cbda80278412db7692f3674b521e0172e8248f5c3be3ff3bc457cd8e5ecb64e", "ground_truth": { "choice": "B" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Two rows\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "2", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-008", "family": "wrapcount", "group_id": "wrapcount-2-dark-1", "image_filename": "wrapcount-2-dark-1.png", "image_sha256": "c4909ee6b6a7455c7db8e81a4e5bef93cab7854991d7afad54cdaaaeb3dc686c", "ground_truth": { "choice": "A" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: Two rows\nB: One row\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "2", "options": [ "2", "1", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-009", "family": "wrapcount", "group_id": "wrapcount-3-light-0", "image_filename": "wrapcount-3-light-0.png", "image_sha256": "b668d6650ec1a07ab46e39d21546acd28abf3067a94981341b313b25e004e851", "ground_truth": { "choice": "B" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: Two rows\nB: Three rows\nC: One row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "3", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-010", "family": "wrapcount", "group_id": "wrapcount-3-light-1", "image_filename": "wrapcount-3-light-1.png", "image_sha256": "f8015e74d11ec40cdd964eb03da44aecf26a02985e94e3f4d59e2ddd8db3df39", "ground_truth": { "choice": "C" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Two rows\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "3", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-011", "family": "wrapcount", "group_id": "wrapcount-3-dark-0", "image_filename": "wrapcount-3-dark-0.png", "image_sha256": "10adae27ebac8e11278eecbfaccd35c3de4cfd63ff5a461dbc98e5e6174d24b9", "ground_truth": { "choice": "C" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Two rows\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "3", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-012", "family": "wrapcount", "group_id": "wrapcount-3-dark-1", "image_filename": "wrapcount-3-dark-1.png", "image_sha256": "1b47a0b631596604ccfe93ebf5f3b23d730064cbc4ff5a9058e18f7d1466dabd", "ground_truth": { "choice": "B" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Three rows\nC: Two rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "3", "options": [ "1", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-001", "family": "wrapalign", "group_id": "wrapalign-left-light-0", "image_filename": "wrapalign-left-light-0.png", "image_sha256": "2581d72463a54c2b3a78c2a211ec00f349c59a01182a1d656c4ef80b17801ef3", "ground_truth": { "choice": "B" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the right inner edge\nB: Against the left inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "right", "left", "center" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-002", "family": "wrapalign", "group_id": "wrapalign-left-light-1", "image_filename": "wrapalign-left-light-1.png", "image_sha256": "1fefa1c83320903e377f47458054c254955bff38821e503f6dbd782067a07837", "ground_truth": { "choice": "C" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Centered, with equal empty space on either side\nB: Against the right inner edge\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "center", "right", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-003", "family": "wrapalign", "group_id": "wrapalign-left-dark-0", "image_filename": "wrapalign-left-dark-0.png", "image_sha256": "287e44f6667d59cb8c85cf4a4da75119a5fe70556912bd058edc0ec36d3d10db", "ground_truth": { "choice": "A" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the left inner edge\nB: Centered, with equal empty space on either side\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "left", "center", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-004", "family": "wrapalign", "group_id": "wrapalign-left-dark-1", "image_filename": "wrapalign-left-dark-1.png", "image_sha256": "895ca53e74780ff9f2db4a7014d254c744e96c604f04852c67e10b5ae2358f04", "ground_truth": { "choice": "A" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the left inner edge\nB: Against the right inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "left", "right", "center" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-005", "family": "wrapalign", "group_id": "wrapalign-center-light-0", "image_filename": "wrapalign-center-light-0.png", "image_sha256": "3cfbcc89320fba4607808bfa17a424c6da3e1ebf4d3193a409b0f67fb61f03a6", "ground_truth": { "choice": "A" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Centered, with equal empty space on either side\nB: Against the right inner edge\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "center", "right", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-006", "family": "wrapalign", "group_id": "wrapalign-center-light-1", "image_filename": "wrapalign-center-light-1.png", "image_sha256": "14f301c90bae0a2f04c1c81aa88cd6e69afe342201096274b1b627daa98e6d0f", "ground_truth": { "choice": "B" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the left inner edge\nB: Centered, with equal empty space on either side\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "left", "center", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-007", "family": "wrapalign", "group_id": "wrapalign-center-dark-0", "image_filename": "wrapalign-center-dark-0.png", "image_sha256": "e215aec67f064c31061b6b9a1b87e2ff18d75e7f92cf40f0cbfe474ca18bfd9a", "ground_truth": { "choice": "B" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the right inner edge\nB: Centered, with equal empty space on either side\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "right", "center", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-008", "family": "wrapalign", "group_id": "wrapalign-center-dark-1", "image_filename": "wrapalign-center-dark-1.png", "image_sha256": "41825df3c3836b765b03940b2125e8bff705eb9efc29c091d680ce1b54f45916", "ground_truth": { "choice": "C" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the left inner edge\nB: Against the right inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "left", "right", "center" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-009", "family": "wrapalign", "group_id": "wrapalign-right-light-0", "image_filename": "wrapalign-right-light-0.png", "image_sha256": "31c0f17c4693d44976136e660a71fdbbe49e08112dfb5f0ae4a9e792e8e86076", "ground_truth": { "choice": "B" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Centered, with equal empty space on either side\nB: Against the right inner edge\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "center", "right", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-010", "family": "wrapalign", "group_id": "wrapalign-right-light-1", "image_filename": "wrapalign-right-light-1.png", "image_sha256": "6256166889394e1725ed5772027cfb2ff6f3bd865044984f95a2dc949f49b616", "ground_truth": { "choice": "C" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Centered, with equal empty space on either side\nB: Against the left inner edge\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "center", "left", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-011", "family": "wrapalign", "group_id": "wrapalign-right-dark-0", "image_filename": "wrapalign-right-dark-0.png", "image_sha256": "52b53c48b26628aaceb090540a6a4cf590a6c34886ba826815bf8cf18ba21208", "ground_truth": { "choice": "C" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the left inner edge\nB: Centered, with equal empty space on either side\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "left", "center", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-012", "family": "wrapalign", "group_id": "wrapalign-right-dark-1", "image_filename": "wrapalign-right-dark-1.png", "image_sha256": "a80216528ef5adb2ac7e9a6b14b045d146618630f8aa4ef53c57255a09780f33", "ground_truth": { "choice": "A" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the right inner edge\nB: Centered, with equal empty space on either side\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "right", "center", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gapcompare-001", "family": "gapcompare", "group_id": "gapcompare-first-light-0", "image_filename": "gapcompare-first-light-0.png", "image_sha256": "cc419375281a36f3a49b6d9072f8b636e4a61d20f6df2e324c32900271406e3a", "ground_truth": { "choice": "C" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The second gap is larger\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "first", "options": [ "equal", "last", "first" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-002", "family": "gapcompare", "group_id": "gapcompare-first-light-1", "image_filename": "gapcompare-first-light-1.png", "image_sha256": "2e49a4e095e3abcfc4c60afb148f03bf6ae4ecd2aee9bd87fa1048bbdeb9f335", "ground_truth": { "choice": "B" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The first gap is larger\nC: The second gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "first", "options": [ "equal", "first", "last" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-003", "family": "gapcompare", "group_id": "gapcompare-first-dark-0", "image_filename": "gapcompare-first-dark-0.png", "image_sha256": "ece00f4c71da4c7da23e19231e904477aefcf1a791afb53334a71b5887ee7d65", "ground_truth": { "choice": "B" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The first gap is larger\nC: The second gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "first", "options": [ "equal", "first", "last" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-004", "family": "gapcompare", "group_id": "gapcompare-first-dark-1", "image_filename": "gapcompare-first-dark-1.png", "image_sha256": "e137bcfefabfc208bd6aabcfcb44a1301b33070dffcd646a323365fe21a842e8", "ground_truth": { "choice": "A" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The first gap is larger\nB: The second gap is larger\nC: The gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "first", "options": [ "first", "last", "equal" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-005", "family": "gapcompare", "group_id": "gapcompare-last-light-0", "image_filename": "gapcompare-last-light-0.png", "image_sha256": "54e5f23e4961afdcbe4a89d5d98726e3afd3e67b8f49ddd8f8ae7ff6de8639f4", "ground_truth": { "choice": "A" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The second gap is larger\nB: The gaps are equal\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "last", "options": [ "last", "equal", "first" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-006", "family": "gapcompare", "group_id": "gapcompare-last-light-1", "image_filename": "gapcompare-last-light-1.png", "image_sha256": "c4c3576b8259241e6f91d9d74fe8c249548ff44fc3a2a22c489e6617f52f2d07", "ground_truth": { "choice": "C" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The first gap is larger\nC: The second gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "last", "options": [ "equal", "first", "last" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-007", "family": "gapcompare", "group_id": "gapcompare-last-dark-0", "image_filename": "gapcompare-last-dark-0.png", "image_sha256": "74e309b66c3c476507138caea045dd8032fa8ec676990f4dd59fc0cf7f6ba09a", "ground_truth": { "choice": "B" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The second gap is larger\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "last", "options": [ "equal", "last", "first" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-008", "family": "gapcompare", "group_id": "gapcompare-last-dark-1", "image_filename": "gapcompare-last-dark-1.png", "image_sha256": "d93a1a8cf7126019bac18613708f8cb50bf094ee9e3b761088a28d8b5e1dd7f4", "ground_truth": { "choice": "C" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The first gap is larger\nB: The gaps are equal\nC: The second gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "last", "options": [ "first", "equal", "last" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-009", "family": "gapcompare", "group_id": "gapcompare-equal-light-0", "image_filename": "gapcompare-equal-light-0.png", "image_sha256": "3a5f7e2741ed276c056812d0d9942666c7ef98bcf6fc6b9da467751543047ccf", "ground_truth": { "choice": "A" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The second gap is larger\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "equal", "last", "first" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-010", "family": "gapcompare", "group_id": "gapcompare-equal-light-1", "image_filename": "gapcompare-equal-light-1.png", "image_sha256": "8256b1f42e57aa77e7f20f68bf3de037068ed496e65d80a22235aa3df55dc585", "ground_truth": { "choice": "A" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The second gap is larger\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "equal", "last", "first" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-011", "family": "gapcompare", "group_id": "gapcompare-equal-dark-0", "image_filename": "gapcompare-equal-dark-0.png", "image_sha256": "a48214a509738ad6006af8ab96f7de94e73ddc76969af1026c7526367ebbeda9", "ground_truth": { "choice": "B" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The second gap is larger\nB: The gaps are equal\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "last", "equal", "first" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-012", "family": "gapcompare", "group_id": "gapcompare-equal-dark-1", "image_filename": "gapcompare-equal-dark-1.png", "image_sha256": "e534c1c57f40c439acac755cc8919d1e7a7af12d6dc1591091adab2a2c650a27", "ground_truth": { "choice": "C" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The second gap is larger\nB: The first gap is larger\nC: The gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "last", "first", "equal" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-padcompare-001", "family": "padcompare", "group_id": "padcompare-horizontal-light-0", "image_filename": "padcompare-horizontal-light-0.png", "image_sha256": "5b296a8ad48fae5b768bb9600813d4fc21f7e93126a111479312b3bb5a04d626", "ground_truth": { "choice": "C" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal and vertical insets are equal\nC: The horizontal inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "horizontal", "options": [ "vertical", "equal", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-002", "family": "padcompare", "group_id": "padcompare-horizontal-light-1", "image_filename": "padcompare-horizontal-light-1.png", "image_sha256": "95ce6788a34ca142b9bb9bb0d99322e25cb680df13310774674cff9a8e3a0132", "ground_truth": { "choice": "A" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The horizontal inset is larger\nB: The horizontal and vertical insets are equal\nC: The vertical inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "horizontal", "options": [ "horizontal", "equal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-003", "family": "padcompare", "group_id": "padcompare-horizontal-dark-0", "image_filename": "padcompare-horizontal-dark-0.png", "image_sha256": "b277e9f0173d3fc4170b0a539af2a0500980de929c094a164aa512c3ee71ffa6", "ground_truth": { "choice": "B" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "horizontal", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-004", "family": "padcompare", "group_id": "padcompare-horizontal-dark-1", "image_filename": "padcompare-horizontal-dark-1.png", "image_sha256": "f4207b17388390034c7a395a612c36f56bc7937d7f6a7e092c10b9a3cc935266", "ground_truth": { "choice": "B" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "horizontal", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-005", "family": "padcompare", "group_id": "padcompare-vertical-light-0", "image_filename": "padcompare-vertical-light-0.png", "image_sha256": "456a4f4f79d28afcce09489c8ec09dfd8ea636271ca39b6031d7e9dd8a15beaf", "ground_truth": { "choice": "A" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal and vertical insets are equal\nC: The horizontal inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "vertical", "options": [ "vertical", "equal", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-006", "family": "padcompare", "group_id": "padcompare-vertical-light-1", "image_filename": "padcompare-vertical-light-1.png", "image_sha256": "acd7ccb49452169ee11251745f7b9861449993832d13e3bf9b758234d3ecb3ff", "ground_truth": { "choice": "C" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The horizontal and vertical insets are equal\nB: The horizontal inset is larger\nC: The vertical inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "vertical", "options": [ "equal", "horizontal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-007", "family": "padcompare", "group_id": "padcompare-vertical-dark-0", "image_filename": "padcompare-vertical-dark-0.png", "image_sha256": "a5b99dab5bcad52f3e412facd7e2f2c95482da7381589baaa4ca8fd39e510916", "ground_truth": { "choice": "A" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "vertical", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-008", "family": "padcompare", "group_id": "padcompare-vertical-dark-1", "image_filename": "padcompare-vertical-dark-1.png", "image_sha256": "a6a85191cf30f3994733c9356d031a968e78e82b8989e6f77e1c43c3cdda825d", "ground_truth": { "choice": "B" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The horizontal inset is larger\nB: The vertical inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "vertical", "options": [ "horizontal", "vertical", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-009", "family": "padcompare", "group_id": "padcompare-equal-light-0", "image_filename": "padcompare-equal-light-0.png", "image_sha256": "1da94e96dd505f78b50f88bdef08b1ee369de72aa7834e1f62f462496ff2f737", "ground_truth": { "choice": "B" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal and vertical insets are equal\nC: The horizontal inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "vertical", "equal", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-010", "family": "padcompare", "group_id": "padcompare-equal-light-1", "image_filename": "padcompare-equal-light-1.png", "image_sha256": "762069b171a8cb02d979c2dbccf6780e96ed4281ff459650617d24547a6d8440", "ground_truth": { "choice": "A" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The horizontal and vertical insets are equal\nB: The horizontal inset is larger\nC: The vertical inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "equal", "horizontal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-011", "family": "padcompare", "group_id": "padcompare-equal-dark-0", "image_filename": "padcompare-equal-dark-0.png", "image_sha256": "6e2ae037b0751829b32e931f8f8ac2283f1189a3e618a794f7c3f6ca40204e13", "ground_truth": { "choice": "C" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-012", "family": "padcompare", "group_id": "padcompare-equal-dark-1", "image_filename": "padcompare-equal-dark-1.png", "image_sha256": "a1d6b43cf6f3cc3d7b51892312f98e040635893623072d8855ef47012de30c97", "ground_truth": { "choice": "C" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The horizontal inset is larger\nB: The vertical inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "horizontal", "vertical", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-blockalign-001", "family": "blockalign", "group_id": "blockalign-left-light-0", "image_filename": "blockalign-left-light-0.png", "image_sha256": "93999cc78a3c0e18d5947e27d1aab602edb9d703fd5dde8b6198e6ebf5331c6f", "ground_truth": { "choice": "A" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the left inner edge\nB: Centered, with equal empty space on either side\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "left", "center", "right" ], "factors": {}, "lines": { "block": [ { "x": 93, "y": 178.015625, "width": 287, "height": 19 }, { "x": 93, "y": 202.8125, "width": 283, "height": 19 }, { "x": 93, "y": 227.609375, "width": 243, "height": 19 }, { "x": 93, "y": 252.40625, "width": 265, "height": 19 }, { "x": 93, "y": 277.203125, "width": 275, "height": 19 }, { "x": 93, "y": 302, "width": 275, "height": 19 }, { "x": 93, "y": 326.796875, "width": 284, "height": 19 }, { "x": 93, "y": 351.59375, "width": 276, "height": 19 }, { "x": 93, "y": 376.390625, "width": 284, "height": 19 }, { "x": 93, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-002", "family": "blockalign", "group_id": "blockalign-left-light-1", "image_filename": "blockalign-left-light-1.png", "image_sha256": "9cbdadaabfaf7d384d94aea382fafd7741085a67e8f8fb3db8999ff1453c07bd", "ground_truth": { "choice": "A" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the left inner edge\nB: Centered, with equal empty space on either side\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "left", "center", "right" ], "factors": {}, "lines": { "block": [ { "x": 100, "y": 190.40625, "width": 291, "height": 19 }, { "x": 130, "y": 215.203125, "width": 261, "height": 19 }, { "x": 93, "y": 240, "width": 298, "height": 19 }, { "x": 127, "y": 264.796875, "width": 264, "height": 19 }, { "x": 141, "y": 289.59375, "width": 250, "height": 19 }, { "x": 98, "y": 314.390625, "width": 293, "height": 19 }, { "x": 102, "y": 339.1875, "width": 289, "height": 19 }, { "x": 135, "y": 363.984375, "width": 256, "height": 19 }, { "x": 131, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-003", "family": "blockalign", "group_id": "blockalign-left-dark-0", "image_filename": "blockalign-left-dark-0.png", "image_sha256": "03da03c4b96165a5e33d3aabd08529c7d210d32d241380a56542c2fa5f5ed98e", "ground_truth": { "choice": "B" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the right inner edge\nB: Against the left inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "right", "left", "center" ], "factors": {}, "lines": { "block": [ { "x": 93, "y": 178.015625, "width": 287, "height": 19 }, { "x": 93, "y": 202.8125, "width": 283, "height": 19 }, { "x": 93, "y": 227.609375, "width": 243, "height": 19 }, { "x": 93, "y": 252.40625, "width": 265, "height": 19 }, { "x": 93, "y": 277.203125, "width": 275, "height": 19 }, { "x": 93, "y": 302, "width": 275, "height": 19 }, { "x": 93, "y": 326.796875, "width": 284, "height": 19 }, { "x": 93, "y": 351.59375, "width": 276, "height": 19 }, { "x": 93, "y": 376.390625, "width": 284, "height": 19 }, { "x": 93, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-004", "family": "blockalign", "group_id": "blockalign-left-dark-1", "image_filename": "blockalign-left-dark-1.png", "image_sha256": "a8da2329c460938a38739b8672030c9a0b81bb9e89cc71d36ed63c9ba51bb447", "ground_truth": { "choice": "C" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Centered, with equal empty space on either side\nB: Against the right inner edge\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "center", "right", "left" ], "factors": {}, "lines": { "block": [ { "x": 100, "y": 190.40625, "width": 291, "height": 19 }, { "x": 130, "y": 215.203125, "width": 261, "height": 19 }, { "x": 93, "y": 240, "width": 298, "height": 19 }, { "x": 127, "y": 264.796875, "width": 264, "height": 19 }, { "x": 141, "y": 289.59375, "width": 250, "height": 19 }, { "x": 98, "y": 314.390625, "width": 293, "height": 19 }, { "x": 102, "y": 339.1875, "width": 289, "height": 19 }, { "x": 135, "y": 363.984375, "width": 256, "height": 19 }, { "x": 131, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-005", "family": "blockalign", "group_id": "blockalign-center-light-0", "image_filename": "blockalign-center-light-0.png", "image_sha256": "c16726c63796edc5b14ff5d8c93c69fd7f9160ee5cf2aac6d56abd2151ca6911", "ground_truth": { "choice": "B" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the right inner edge\nB: Centered, with equal empty space on either side\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "right", "center", "left" ], "factors": {}, "lines": { "block": [ { "x": 251, "y": 178.015625, "width": 287, "height": 19 }, { "x": 251, "y": 202.8125, "width": 283, "height": 19 }, { "x": 251, "y": 227.609375, "width": 243, "height": 19 }, { "x": 251, "y": 252.40625, "width": 265, "height": 19 }, { "x": 251, "y": 277.203125, "width": 275, "height": 19 }, { "x": 251, "y": 302, "width": 275, "height": 19 }, { "x": 251, "y": 326.796875, "width": 284, "height": 19 }, { "x": 251, "y": 351.59375, "width": 276, "height": 19 }, { "x": 251, "y": 376.390625, "width": 284, "height": 19 }, { "x": 251, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-006", "family": "blockalign", "group_id": "blockalign-center-light-1", "image_filename": "blockalign-center-light-1.png", "image_sha256": "c1d2ea027ca7c5544c3228e0e7ef3c1b763372f0e5abc3d6f8b8e03feac01fd2", "ground_truth": { "choice": "C" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the right inner edge\nB: Against the left inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "right", "left", "center" ], "factors": {}, "lines": { "block": [ { "x": 258, "y": 190.40625, "width": 291, "height": 19 }, { "x": 288, "y": 215.203125, "width": 261, "height": 19 }, { "x": 251, "y": 240, "width": 298, "height": 19 }, { "x": 285, "y": 264.796875, "width": 264, "height": 19 }, { "x": 299, "y": 289.59375, "width": 250, "height": 19 }, { "x": 256, "y": 314.390625, "width": 293, "height": 19 }, { "x": 260, "y": 339.1875, "width": 289, "height": 19 }, { "x": 293, "y": 363.984375, "width": 256, "height": 19 }, { "x": 289, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-007", "family": "blockalign", "group_id": "blockalign-center-dark-0", "image_filename": "blockalign-center-dark-0.png", "image_sha256": "009f707473ba4f5e85536576eea3c4f35b761c70440f013cf9038979f8469288", "ground_truth": { "choice": "A" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Centered, with equal empty space on either side\nB: Against the left inner edge\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "center", "left", "right" ], "factors": {}, "lines": { "block": [ { "x": 251, "y": 178.015625, "width": 287, "height": 19 }, { "x": 251, "y": 202.8125, "width": 283, "height": 19 }, { "x": 251, "y": 227.609375, "width": 243, "height": 19 }, { "x": 251, "y": 252.40625, "width": 265, "height": 19 }, { "x": 251, "y": 277.203125, "width": 275, "height": 19 }, { "x": 251, "y": 302, "width": 275, "height": 19 }, { "x": 251, "y": 326.796875, "width": 284, "height": 19 }, { "x": 251, "y": 351.59375, "width": 276, "height": 19 }, { "x": 251, "y": 376.390625, "width": 284, "height": 19 }, { "x": 251, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-008", "family": "blockalign", "group_id": "blockalign-center-dark-1", "image_filename": "blockalign-center-dark-1.png", "image_sha256": "ba60d8ec38eaff61c59a1dc8310d86bde2a0541f1b9068137d1e622d1bbd3996", "ground_truth": { "choice": "C" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the right inner edge\nB: Against the left inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "right", "left", "center" ], "factors": {}, "lines": { "block": [ { "x": 258, "y": 190.40625, "width": 291, "height": 19 }, { "x": 288, "y": 215.203125, "width": 261, "height": 19 }, { "x": 251, "y": 240, "width": 298, "height": 19 }, { "x": 285, "y": 264.796875, "width": 264, "height": 19 }, { "x": 299, "y": 289.59375, "width": 250, "height": 19 }, { "x": 256, "y": 314.390625, "width": 293, "height": 19 }, { "x": 260, "y": 339.1875, "width": 289, "height": 19 }, { "x": 293, "y": 363.984375, "width": 256, "height": 19 }, { "x": 289, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-009", "family": "blockalign", "group_id": "blockalign-right-light-0", "image_filename": "blockalign-right-light-0.png", "image_sha256": "d4c23856fcf5b4b04e2d3b939e7d4117d75a41b3f1c01b96444692528c135684", "ground_truth": { "choice": "A" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the right inner edge\nB: Against the left inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "right", "left", "center" ], "factors": {}, "lines": { "block": [ { "x": 409, "y": 178.015625, "width": 287, "height": 19 }, { "x": 409, "y": 202.8125, "width": 283, "height": 19 }, { "x": 409, "y": 227.609375, "width": 243, "height": 19 }, { "x": 409, "y": 252.40625, "width": 265, "height": 19 }, { "x": 409, "y": 277.203125, "width": 275, "height": 19 }, { "x": 409, "y": 302, "width": 275, "height": 19 }, { "x": 409, "y": 326.796875, "width": 284, "height": 19 }, { "x": 409, "y": 351.59375, "width": 276, "height": 19 }, { "x": 409, "y": 376.390625, "width": 284, "height": 19 }, { "x": 409, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-010", "family": "blockalign", "group_id": "blockalign-right-light-1", "image_filename": "blockalign-right-light-1.png", "image_sha256": "58d73942b4c7479d04a47fbcda3c873327a521a91cfb09cdb589dc9394711ed0", "ground_truth": { "choice": "B" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the left inner edge\nB: Against the right inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "left", "right", "center" ], "factors": {}, "lines": { "block": [ { "x": 416, "y": 190.40625, "width": 291, "height": 19 }, { "x": 446, "y": 215.203125, "width": 261, "height": 19 }, { "x": 409, "y": 240, "width": 298, "height": 19 }, { "x": 443, "y": 264.796875, "width": 264, "height": 19 }, { "x": 457, "y": 289.59375, "width": 250, "height": 19 }, { "x": 414, "y": 314.390625, "width": 293, "height": 19 }, { "x": 418, "y": 339.1875, "width": 289, "height": 19 }, { "x": 451, "y": 363.984375, "width": 256, "height": 19 }, { "x": 447, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-011", "family": "blockalign", "group_id": "blockalign-right-dark-0", "image_filename": "blockalign-right-dark-0.png", "image_sha256": "f34c2f431c1a1ae5e21bdc12bda54a4ff809d38515b4627acdc2caabfae9bfbf", "ground_truth": { "choice": "C" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Centered, with equal empty space on either side\nB: Against the left inner edge\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "center", "left", "right" ], "factors": {}, "lines": { "block": [ { "x": 409, "y": 178.015625, "width": 287, "height": 19 }, { "x": 409, "y": 202.8125, "width": 283, "height": 19 }, { "x": 409, "y": 227.609375, "width": 243, "height": 19 }, { "x": 409, "y": 252.40625, "width": 265, "height": 19 }, { "x": 409, "y": 277.203125, "width": 275, "height": 19 }, { "x": 409, "y": 302, "width": 275, "height": 19 }, { "x": 409, "y": 326.796875, "width": 284, "height": 19 }, { "x": 409, "y": 351.59375, "width": 276, "height": 19 }, { "x": 409, "y": 376.390625, "width": 284, "height": 19 }, { "x": 409, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-012", "family": "blockalign", "group_id": "blockalign-right-dark-1", "image_filename": "blockalign-right-dark-1.png", "image_sha256": "2afc47f7c1b07f0c593f365a0f9b7a03cf87bdac25a2574b857e1313c1a8ab8f", "ground_truth": { "choice": "B" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Centered, with equal empty space on either side\nB: Against the right inner edge\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "center", "right", "left" ], "factors": {}, "lines": { "block": [ { "x": 416, "y": 190.40625, "width": 291, "height": 19 }, { "x": 446, "y": 215.203125, "width": 261, "height": 19 }, { "x": 409, "y": 240, "width": 298, "height": 19 }, { "x": 443, "y": 264.796875, "width": 264, "height": 19 }, { "x": 457, "y": 289.59375, "width": 250, "height": 19 }, { "x": 414, "y": 314.390625, "width": 293, "height": 19 }, { "x": 418, "y": 339.1875, "width": 289, "height": 19 }, { "x": 451, "y": 363.984375, "width": 256, "height": 19 }, { "x": 447, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-widthcompare-001", "family": "widthcompare", "group_id": "widthcompare-first-light-0", "image_filename": "widthcompare-first-light-0.png", "image_sha256": "46860f6ee36c88d21df117584a6a7d1bee3dbbdddde639fd12a2c34a6689826c", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: Their widths are equal\nB: The left panel is wider\nC: The right panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "first", "options": [ "equal", "first", "last" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-002", "family": "widthcompare", "group_id": "widthcompare-first-light-1", "image_filename": "widthcompare-first-light-1.png", "image_sha256": "ba75b11311674a94c11f44abb0d3ff347d0f39a52d78d2f39b3a82d6e5a79a42", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: Their widths are equal\nC: The left panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "first", "options": [ "last", "equal", "first" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-003", "family": "widthcompare", "group_id": "widthcompare-first-dark-0", "image_filename": "widthcompare-first-dark-0.png", "image_sha256": "aa7026825d6facf9b94cefd1b70fa35bdca85928c7185f0a30027647b969fdc1", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The left panel is wider\nB: The right panel is wider\nC: Their widths are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "first", "options": [ "first", "last", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-004", "family": "widthcompare", "group_id": "widthcompare-first-dark-1", "image_filename": "widthcompare-first-dark-1.png", "image_sha256": "2744b96c2c1184f933a8b49483327c6690744736058300c3d8404d8cf5435183", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: Their widths are equal\nB: The left panel is wider\nC: The right panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "first", "options": [ "equal", "first", "last" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-005", "family": "widthcompare", "group_id": "widthcompare-last-light-0", "image_filename": "widthcompare-last-light-0.png", "image_sha256": "fd61274910c02eb937ec1bb8f30b5cc754d5da33c564585598acceee3074c616", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The left panel is wider\nB: The right panel is wider\nC: Their widths are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "last", "options": [ "first", "last", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-006", "family": "widthcompare", "group_id": "widthcompare-last-light-1", "image_filename": "widthcompare-last-light-1.png", "image_sha256": "ff0d815424a947f8f6975d2aad9c9fd96d3709c09822ac70db434a8f8d1d6e64", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: The left panel is wider\nC: Their widths are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "last", "options": [ "last", "first", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-007", "family": "widthcompare", "group_id": "widthcompare-last-dark-0", "image_filename": "widthcompare-last-dark-0.png", "image_sha256": "898d6547a4e05b4b219a2c121ad66a3546a5fcbba53d03419b7c97fa84bca25d", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: Their widths are equal\nB: The left panel is wider\nC: The right panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "last", "options": [ "equal", "first", "last" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-008", "family": "widthcompare", "group_id": "widthcompare-last-dark-1", "image_filename": "widthcompare-last-dark-1.png", "image_sha256": "1cf129e23880d6e00dcab320ef154e9e9762c9d86e4c56842e04fd1d19419a8a", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: Their widths are equal\nC: The left panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "last", "options": [ "last", "equal", "first" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-009", "family": "widthcompare", "group_id": "widthcompare-equal-light-0", "image_filename": "widthcompare-equal-light-0.png", "image_sha256": "2d59ff6eb852db55988086f0037c1a1815945900c24d37930b5271755553c87a", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: Their widths are equal\nC: The left panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "last", "equal", "first" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-010", "family": "widthcompare", "group_id": "widthcompare-equal-light-1", "image_filename": "widthcompare-equal-light-1.png", "image_sha256": "cc3219943ba440fefa67f513a731f5112c14d3b2a8ea874d26f3835e5e4ed08b", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: Their widths are equal\nB: The left panel is wider\nC: The right panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "equal", "first", "last" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-011", "family": "widthcompare", "group_id": "widthcompare-equal-dark-0", "image_filename": "widthcompare-equal-dark-0.png", "image_sha256": "87e943ef52547d0aa4da2c6c86e801853d7a2c5d19215f84741e040b00eaef20", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: The left panel is wider\nC: Their widths are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "last", "first", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-012", "family": "widthcompare", "group_id": "widthcompare-equal-dark-1", "image_filename": "widthcompare-equal-dark-1.png", "image_sha256": "56f921860765116da9a904be6c93172ae73204166e028a79897a6b609210e801", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: The left panel is wider\nC: Their widths are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "last", "first", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-001", "family": "groupgap", "group_id": "groupgap-between-light-0", "image_filename": "groupgap-between-light-0.png", "image_sha256": "ceb82f5b741bb1c82eeecadad888a8a37634bdfcd13b518fd090579d29d3f722", "ground_truth": { "choice": "B" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group and within-group gaps are equal\nB: The between-group gap is larger\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "between", "options": [ "equal", "between", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-002", "family": "groupgap", "group_id": "groupgap-between-light-1", "image_filename": "groupgap-between-light-1.png", "image_sha256": "d89e016a56363e65ec619535629b869fc91ad54552753715f0ee63b7e8f98baa", "ground_truth": { "choice": "B" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The within-group gaps are larger\nB: The between-group gap is larger\nC: The between-group and within-group gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "between", "options": [ "within", "between", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-003", "family": "groupgap", "group_id": "groupgap-between-dark-0", "image_filename": "groupgap-between-dark-0.png", "image_sha256": "70bb1bfdfa598530782f35f113acef2c91778b67a15b798d32679c8f73533ec4", "ground_truth": { "choice": "B" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The within-group gaps are larger\nB: The between-group gap is larger\nC: The between-group and within-group gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "between", "options": [ "within", "between", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-004", "family": "groupgap", "group_id": "groupgap-between-dark-1", "image_filename": "groupgap-between-dark-1.png", "image_sha256": "c6a918cb13023fed221b6ebe3f6b8d2bc0884e5a9cc46b9222eeee4a3560b66b", "ground_truth": { "choice": "A" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group gap is larger\nB: The between-group and within-group gaps are equal\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "between", "options": [ "between", "equal", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-005", "family": "groupgap", "group_id": "groupgap-within-light-0", "image_filename": "groupgap-within-light-0.png", "image_sha256": "d6b8b3fb38cdca6a7808c902b5c8d76e8bcb1b139ffdcf5d4bc6b0b18fb6723d", "ground_truth": { "choice": "C" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group gap is larger\nB: The between-group and within-group gaps are equal\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "within", "options": [ "between", "equal", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-006", "family": "groupgap", "group_id": "groupgap-within-light-1", "image_filename": "groupgap-within-light-1.png", "image_sha256": "012c7b0c080a5dbf52ef141cbadb2baf491c0a1ca10ad5f08c06074b104afc7a", "ground_truth": { "choice": "A" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The within-group gaps are larger\nB: The between-group gap is larger\nC: The between-group and within-group gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "within", "options": [ "within", "between", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-007", "family": "groupgap", "group_id": "groupgap-within-dark-0", "image_filename": "groupgap-within-dark-0.png", "image_sha256": "aa29f9546ddc83ca8a3072b2c656282e867da95f84e38d7c5605c639ee2c8592", "ground_truth": { "choice": "C" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group gap is larger\nB: The between-group and within-group gaps are equal\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "within", "options": [ "between", "equal", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-008", "family": "groupgap", "group_id": "groupgap-within-dark-1", "image_filename": "groupgap-within-dark-1.png", "image_sha256": "c032eff2522bf2eb2c35de95e00a4acef8d668150b4afbc593c49ee6f27c6f59", "ground_truth": { "choice": "C" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group and within-group gaps are equal\nB: The between-group gap is larger\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "within", "options": [ "equal", "between", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-009", "family": "groupgap", "group_id": "groupgap-equal-light-0", "image_filename": "groupgap-equal-light-0.png", "image_sha256": "324fb72a105e935b01471b580b3e9acc2cfa26b3f44664ef57e61f3e8ecd0a2b", "ground_truth": { "choice": "B" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The within-group gaps are larger\nB: The between-group and within-group gaps are equal\nC: The between-group gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "within", "equal", "between" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-010", "family": "groupgap", "group_id": "groupgap-equal-light-1", "image_filename": "groupgap-equal-light-1.png", "image_sha256": "a966dcb73dc3edbb12a3a4ab4b5704bbf7a00a2e9f98f6da52213be89f9ede84", "ground_truth": { "choice": "A" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group and within-group gaps are equal\nB: The between-group gap is larger\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "equal", "between", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-011", "family": "groupgap", "group_id": "groupgap-equal-dark-0", "image_filename": "groupgap-equal-dark-0.png", "image_sha256": "10450b9a549a8d0c0c1e980fdf8564ee520b408eb1d1da67f729fec64bea7f41", "ground_truth": { "choice": "A" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group and within-group gaps are equal\nB: The between-group gap is larger\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "equal", "between", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-012", "family": "groupgap", "group_id": "groupgap-equal-dark-1", "image_filename": "groupgap-equal-dark-1.png", "image_sha256": "09e520206c0615b65a9ac6a5325091542c1eae2b92f556e98542028f188524a2", "ground_truth": { "choice": "C" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The within-group gaps are larger\nB: The between-group gap is larger\nC: The between-group and within-group gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "within", "between", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-topflow-001", "family": "topflow", "group_id": "h-01", "image_filename": "h-01.png", "image_sha256": "741b99f72d76efde3e95c36ef17639ab23de30d920877562d55f355743f9a36e", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-001", "family": "nestedflow", "group_id": "h-01", "image_filename": "h-01.png", "image_sha256": "741b99f72d76efde3e95c36ef17639ab23de30d920877562d55f355743f9a36e", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One horizontal row\nB: One vertical column\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "row", "column", "wrapped" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-002", "family": "topflow", "group_id": "h-02", "image_filename": "h-02.png", "image_sha256": "ce9bdffee41108c8564bbae34842327d1c77448ad21c047a46f0f1eb83ad5afe", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-002", "family": "nestedflow", "group_id": "h-02", "image_filename": "h-02.png", "image_sha256": "ce9bdffee41108c8564bbae34842327d1c77448ad21c047a46f0f1eb83ad5afe", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One vertical column\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "wrapped", "column", "row" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-003", "family": "topflow", "group_id": "h-03", "image_filename": "h-03.png", "image_sha256": "b5012b0f72f8c622150f264bba9b29e3987c9950f49fca02dd53e6b86335c785", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-003", "family": "nestedflow", "group_id": "h-03", "image_filename": "h-03.png", "image_sha256": "b5012b0f72f8c622150f264bba9b29e3987c9950f49fca02dd53e6b86335c785", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-004", "family": "topflow", "group_id": "h-04", "image_filename": "h-04.png", "image_sha256": "ee6e41ee6423d82929e6b2792f099c30e41e9b4f093a59891f3dff64d49519a1", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "column", "row" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-004", "family": "nestedflow", "group_id": "h-04", "image_filename": "h-04.png", "image_sha256": "ee6e41ee6423d82929e6b2792f099c30e41e9b4f093a59891f3dff64d49519a1", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: Multiple horizontal rows, with a shorter final row\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "column", "wrapped", "row" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-005", "family": "topflow", "group_id": "h-05", "image_filename": "h-05.png", "image_sha256": "b8553aa6a3374f08b6ac80b8dd5164e6d34ead6b3cd6507ee7890d9231eae4a0", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "column", "row" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-005", "family": "nestedflow", "group_id": "h-05", "image_filename": "h-05.png", "image_sha256": "b8553aa6a3374f08b6ac80b8dd5164e6d34ead6b3cd6507ee7890d9231eae4a0", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One horizontal row\nB: Multiple horizontal rows, with a shorter final row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "row", "wrapped", "column" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-006", "family": "topflow", "group_id": "h-06", "image_filename": "h-06.png", "image_sha256": "dead20b06b2b84a2e55ee13e0ae10470d565bc40d3183bf46215687a91046935", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-006", "family": "nestedflow", "group_id": "h-06", "image_filename": "h-06.png", "image_sha256": "dead20b06b2b84a2e55ee13e0ae10470d565bc40d3183bf46215687a91046935", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One horizontal row\nB: One vertical column\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "row", "column", "wrapped" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-007", "family": "topflow", "group_id": "h-07", "image_filename": "h-07.png", "image_sha256": "deb9c818c6b4cd55dd8f62920c258e5e4e50c69aa5ca326621242f7464d0e3d3", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-007", "family": "nestedflow", "group_id": "h-07", "image_filename": "h-07.png", "image_sha256": "deb9c818c6b4cd55dd8f62920c258e5e4e50c69aa5ca326621242f7464d0e3d3", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-008", "family": "topflow", "group_id": "h-08", "image_filename": "h-08.png", "image_sha256": "ce7968612ee74659b54fe1e40ac2eb572c8db02c4ad660a30af89a5a762aec4d", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "column", "row" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-008", "family": "nestedflow", "group_id": "h-08", "image_filename": "h-08.png", "image_sha256": "ce7968612ee74659b54fe1e40ac2eb572c8db02c4ad660a30af89a5a762aec4d", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: Multiple horizontal rows, with a shorter final row\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "column", "wrapped", "row" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-009", "family": "topflow", "group_id": "h-09", "image_filename": "h-09.png", "image_sha256": "882f6b21e43601e586930da4bae5ac5ceb3173a0854b23000f9184b87a6a71df", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-009", "family": "nestedflow", "group_id": "h-09", "image_filename": "h-09.png", "image_sha256": "882f6b21e43601e586930da4bae5ac5ceb3173a0854b23000f9184b87a6a71df", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: Multiple horizontal rows, with a shorter final row\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "wrapped", "options": [ "column", "wrapped", "row" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-001", "family": "nestedwrap", "group_id": "h-09", "image_filename": "h-09.png", "image_sha256": "882f6b21e43601e586930da4bae5ac5ceb3173a0854b23000f9184b87a6a71df", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's right inner edge\nB: Against that frame's left inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "right", "left", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-010", "family": "topflow", "group_id": "h-10", "image_filename": "h-10.png", "image_sha256": "38331671e2ad6b023b8c85ebfed1a00bfcf4e065af92fc235e80bce6eae12ec6", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-010", "family": "nestedflow", "group_id": "h-10", "image_filename": "h-10.png", "image_sha256": "38331671e2ad6b023b8c85ebfed1a00bfcf4e065af92fc235e80bce6eae12ec6", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: Multiple horizontal rows, with a shorter final row\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "wrapped", "options": [ "column", "wrapped", "row" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-002", "family": "nestedwrap", "group_id": "h-10", "image_filename": "h-10.png", "image_sha256": "38331671e2ad6b023b8c85ebfed1a00bfcf4e065af92fc235e80bce6eae12ec6", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's right inner edge\nB: Against that frame's left inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "right", "left", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-011", "family": "topflow", "group_id": "h-11", "image_filename": "h-11.png", "image_sha256": "11fab0be75f11a84e4ce883389ccfd4ad1de5a331d43fe5f2125483fb75c0414", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "column", "row" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-011", "family": "nestedflow", "group_id": "h-11", "image_filename": "h-11.png", "image_sha256": "11fab0be75f11a84e4ce883389ccfd4ad1de5a331d43fe5f2125483fb75c0414", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "wrapped", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-003", "family": "nestedwrap", "group_id": "h-11", "image_filename": "h-11.png", "image_sha256": "11fab0be75f11a84e4ce883389ccfd4ad1de5a331d43fe5f2125483fb75c0414", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's right inner edge\nB: Centered in that frame\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "right", "center", "left" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-012", "family": "topflow", "group_id": "h-12", "image_filename": "h-12.png", "image_sha256": "ce42e30c0bffa72ca47a923d41027233a6c7d1f57dd0e861e7e55d5c45b37e7c", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-012", "family": "nestedflow", "group_id": "h-12", "image_filename": "h-12.png", "image_sha256": "ce42e30c0bffa72ca47a923d41027233a6c7d1f57dd0e861e7e55d5c45b37e7c", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One vertical column\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "wrapped", "options": [ "wrapped", "column", "row" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-004", "family": "nestedwrap", "group_id": "h-12", "image_filename": "h-12.png", "image_sha256": "ce42e30c0bffa72ca47a923d41027233a6c7d1f57dd0e861e7e55d5c45b37e7c", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "left", "right", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-013", "family": "topflow", "group_id": "h-13", "image_filename": "h-13.png", "image_sha256": "73cc1b0894680664693d0cd4727ba8fbf7e28c29dd8826cd4bf67e92b220a67e", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "column", "row" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-013", "family": "nestedflow", "group_id": "h-13", "image_filename": "h-13.png", "image_sha256": "73cc1b0894680664693d0cd4727ba8fbf7e28c29dd8826cd4bf67e92b220a67e", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One horizontal row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "wrapped", "row", "column" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-014", "family": "topflow", "group_id": "h-14", "image_filename": "h-14.png", "image_sha256": "b9c75a3dc7e8f5176ebc4a845e41fe06c3e91bdc3fdb02ff91cb6cf8bd63dc8f", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-014", "family": "nestedflow", "group_id": "h-14", "image_filename": "h-14.png", "image_sha256": "b9c75a3dc7e8f5176ebc4a845e41fe06c3e91bdc3fdb02ff91cb6cf8bd63dc8f", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One vertical column\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "wrapped", "column", "row" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-015", "family": "topflow", "group_id": "h-15", "image_filename": "h-15.png", "image_sha256": "2a0d01a2f64b8d14df5ce81eaf53efb7422372ae9757687eb7f14ea7d7bc6916", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-015", "family": "nestedflow", "group_id": "h-15", "image_filename": "h-15.png", "image_sha256": "2a0d01a2f64b8d14df5ce81eaf53efb7422372ae9757687eb7f14ea7d7bc6916", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One vertical column\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "wrapped", "column", "row" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-016", "family": "topflow", "group_id": "h-16", "image_filename": "h-16.png", "image_sha256": "81202a6843a545ef0629321d9309200371f961552c69b24027c4136e5c77e49a", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "column", "row" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-016", "family": "nestedflow", "group_id": "h-16", "image_filename": "h-16.png", "image_sha256": "81202a6843a545ef0629321d9309200371f961552c69b24027c4136e5c77e49a", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-017", "family": "topflow", "group_id": "h-17", "image_filename": "h-17.png", "image_sha256": "a6dde0c147d50a2fb7a2347d51e9e3351dde82f2e501be2b338b4ceeaad6883f", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-017", "family": "nestedflow", "group_id": "h-17", "image_filename": "h-17.png", "image_sha256": "a6dde0c147d50a2fb7a2347d51e9e3351dde82f2e501be2b338b4ceeaad6883f", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-018", "family": "topflow", "group_id": "h-18", "image_filename": "h-18.png", "image_sha256": "d0bbda389e76accbcb4c2d62e9931cca31806af19f961963f3a5f618ced07cb8", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-018", "family": "nestedflow", "group_id": "h-18", "image_filename": "h-18.png", "image_sha256": "d0bbda389e76accbcb4c2d62e9931cca31806af19f961963f3a5f618ced07cb8", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One horizontal row\nB: Multiple horizontal rows, with a shorter final row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "row", "wrapped", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-019", "family": "topflow", "group_id": "h-19", "image_filename": "h-19.png", "image_sha256": "747f3f9149d4ec8088138a66363ee8c3ca1a73699c6e0ec0bc06930070b028be", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-019", "family": "nestedflow", "group_id": "h-19", "image_filename": "h-19.png", "image_sha256": "747f3f9149d4ec8088138a66363ee8c3ca1a73699c6e0ec0bc06930070b028be", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One horizontal row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "wrapped", "row", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-020", "family": "topflow", "group_id": "h-20", "image_filename": "h-20.png", "image_sha256": "c206898a1cce5a359c56a498bbf996ed59750d7550277738befcbc065e071678", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-020", "family": "nestedflow", "group_id": "h-20", "image_filename": "h-20.png", "image_sha256": "c206898a1cce5a359c56a498bbf996ed59750d7550277738befcbc065e071678", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-021", "family": "topflow", "group_id": "h-21", "image_filename": "h-21.png", "image_sha256": "1d4800c3ed8fc4fed51234e11ff9e69e776daf5d7e63e0dd8973bdcffd70fb26", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "column", "row" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-021", "family": "nestedflow", "group_id": "h-21", "image_filename": "h-21.png", "image_sha256": "1d4800c3ed8fc4fed51234e11ff9e69e776daf5d7e63e0dd8973bdcffd70fb26", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: Multiple horizontal rows, with a shorter final row\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "wrapped", "options": [ "column", "wrapped", "row" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-005", "family": "nestedwrap", "group_id": "h-21", "image_filename": "h-21.png", "image_sha256": "1d4800c3ed8fc4fed51234e11ff9e69e776daf5d7e63e0dd8973bdcffd70fb26", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Centered in that frame\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "left", "center", "right" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-022", "family": "topflow", "group_id": "h-22", "image_filename": "h-22.png", "image_sha256": "a7b0cd9eb4f906fa4967205ed5db6e0c7e71282dd1312c6b47690b8aa501cfa9", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-022", "family": "nestedflow", "group_id": "h-22", "image_filename": "h-22.png", "image_sha256": "a7b0cd9eb4f906fa4967205ed5db6e0c7e71282dd1312c6b47690b8aa501cfa9", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One vertical column\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "wrapped", "options": [ "wrapped", "column", "row" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-006", "family": "nestedwrap", "group_id": "h-22", "image_filename": "h-22.png", "image_sha256": "a7b0cd9eb4f906fa4967205ed5db6e0c7e71282dd1312c6b47690b8aa501cfa9", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's right inner edge\nB: Against that frame's left inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "right", "left", "center" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-023", "family": "topflow", "group_id": "h-23", "image_filename": "h-23.png", "image_sha256": "6047742c65fc42dcbf3ff66b9a3f2d8c60e28fdb3910e0ff99529f204c2de7ff", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "column", "row" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-023", "family": "nestedflow", "group_id": "h-23", "image_filename": "h-23.png", "image_sha256": "6047742c65fc42dcbf3ff66b9a3f2d8c60e28fdb3910e0ff99529f204c2de7ff", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One horizontal row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "wrapped", "options": [ "wrapped", "row", "column" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-007", "family": "nestedwrap", "group_id": "h-23", "image_filename": "h-23.png", "image_sha256": "6047742c65fc42dcbf3ff66b9a3f2d8c60e28fdb3910e0ff99529f204c2de7ff", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Centered in that frame\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "left", "center", "right" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-024", "family": "topflow", "group_id": "h-24", "image_filename": "h-24.png", "image_sha256": "78b83f2cb2d0ed8ba47fc3705d2d914b8daabf05c5775d59531abcff06fce754", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-024", "family": "nestedflow", "group_id": "h-24", "image_filename": "h-24.png", "image_sha256": "78b83f2cb2d0ed8ba47fc3705d2d914b8daabf05c5775d59531abcff06fce754", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One horizontal row\nB: Multiple horizontal rows, with a shorter final row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "wrapped", "options": [ "row", "wrapped", "column" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-008", "family": "nestedwrap", "group_id": "h-24", "image_filename": "h-24.png", "image_sha256": "78b83f2cb2d0ed8ba47fc3705d2d914b8daabf05c5775d59531abcff06fce754", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "center", "right", "left" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-009", "family": "nestedwrap", "group_id": "h-25", "image_filename": "h-25.png", "image_sha256": "27855b2e6fced92816b6fb010b410ab15744b572d561dd18cb2a672e49939fbe", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "left", "right", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-010", "family": "nestedwrap", "group_id": "h-26", "image_filename": "h-26.png", "image_sha256": "0286d84e3f7b26d0056ade80f04cea37c03f43c2806c0972e3f58fd7d010635b", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's left inner edge\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "center", "left", "right" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-011", "family": "nestedwrap", "group_id": "h-27", "image_filename": "h-27.png", "image_sha256": "d6307216da29e0a4f2bf1f2c84210ded0dba685ea928332720cf5ab92cf79e31", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's left inner edge\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "center", "left", "right" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-012", "family": "nestedwrap", "group_id": "h-28", "image_filename": "h-28.png", "image_sha256": "544e484f4d9c02418e4406dd6c79ad4ab511897f3d55f9848531250a717c6755", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "left", "right", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-013", "family": "nestedwrap", "group_id": "h-29", "image_filename": "h-29.png", "image_sha256": "bf9a6f99ad42c19b670a7394e559760fd81c6052e8717afdd345a0b1a1e79202", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "center", "right", "left" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-014", "family": "nestedwrap", "group_id": "h-30", "image_filename": "h-30.png", "image_sha256": "5efdb664d144ae4df5c7dc476be71682041956693b29cb6c9d303f35bbaf20f9", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Centered in that frame\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "left", "center", "right" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-015", "family": "nestedwrap", "group_id": "h-31", "image_filename": "h-31.png", "image_sha256": "7d477ca9e59a35587d4499445de2aec0a1ee19fb7a3bee4cb7c398271db3bb5e", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "left", "right", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-016", "family": "nestedwrap", "group_id": "h-32", "image_filename": "h-32.png", "image_sha256": "eae65c052f264b8402769a5300480339f38fab9bcddc4b88f221d157a2680e1a", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "left", "right", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-017", "family": "nestedwrap", "group_id": "h-33", "image_filename": "h-33.png", "image_sha256": "a0fdde798f6344c8ff03337e534ae7467ea10432fabc306bcf4b3040fe978b28", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "center", "right", "left" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-018", "family": "nestedwrap", "group_id": "h-34", "image_filename": "h-34.png", "image_sha256": "a7b0142776ab45bd9f68c91aac9951adb73f83d76af73357c62636ba25023da7", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "center", "right", "left" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-019", "family": "nestedwrap", "group_id": "h-35", "image_filename": "h-35.png", "image_sha256": "97dd5a262593af54f5ef693d1ec37feb67bf4900ab404d1828b3e4e94467fb39", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's right inner edge\nB: Against that frame's left inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "right", "left", "center" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-020", "family": "nestedwrap", "group_id": "h-36", "image_filename": "h-36.png", "image_sha256": "175d95b20cafef3137e69ede0ef0c8a19d5274444584930d3157cfe571c20a35", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "center", "right", "left" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-021", "family": "nestedwrap", "group_id": "h-37", "image_filename": "h-37.png", "image_sha256": "7feb231c6385acf9349d6a1d4ad713c28886b149c9ae395e4bc3c63d6c8bda0b", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Centered in that frame\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "left", "center", "right" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-022", "family": "nestedwrap", "group_id": "h-38", "image_filename": "h-38.png", "image_sha256": "373d52b50a1bb6d35560a59d1f4b2c687bc040177d8d76171ad1ba0ef1f392af", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "left", "right", "center" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-023", "family": "nestedwrap", "group_id": "h-39", "image_filename": "h-39.png", "image_sha256": "8ba7648a28cac3e204f80cfd1748fad8e60cdf0697e98b688ad73f96d71df193", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "center", "right", "left" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-024", "family": "nestedwrap", "group_id": "h-40", "image_filename": "h-40.png", "image_sha256": "a4efeff5aef04660008577c6a8bef60f07d4d801cca4eb52a973313e4c936b56", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's left inner edge\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "center", "left", "right" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "right" }, "lines": {} } } ], "results": [ { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "5407cb7b-4436-47bf-bed0-f9b5a72b072f", "result_sha256": "86bea84a9db653ac6ea14dbfa9ed060cbb3f3fe47947f91ad474360199e3f88e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 6.207112283911556, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:21.405667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "10730dd5-006d-48fc-a1f9-7325a111c0b0", "result_sha256": "b72f6309f1594b9877c947c2f68ee0621df3daa4a95cafc2e08d58e72ad9b2c8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 6.003187703900039, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:17.563848+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "5c0e8ec8-741b-45cf-b732-af708abe3c53", "result_sha256": "3731d6e5033184611de887b9df32ecfbb9b44c3aed2030852a3cab192ee75498", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.535188506008126, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:09.322044+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d56e281c-7fb8-408f-94ef-648895cecafd", "result_sha256": "eff208d3c4b20a3f22f12ff81a4b63e1e666beb4a1ecf23e0a005c9444370387", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.772584279999137, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:11.712381+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "7dc99623-fe96-434a-86f9-4378826d51eb", "result_sha256": "5ee17f32999550ca20cffc0215115db30e358580a952d083a22607320b216592", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 4.451809523976408, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:38.464886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "24a33e84-b3a4-4f3f-a65e-ea79f8671515", "result_sha256": "34637cd47428d52d4debac4da39abf0eb11c20d18e903742dfa86c45b94010d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.4140214830404148, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:50.340118+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4855c266-f170-49da-807f-40d06d24bbc0", "result_sha256": "8815de9fa2e1b2263a2ad5f43dbfad2912c2f83d0671f539ec0d7e43ab120e76", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 4.36707341496367, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:40.962305+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d5c66013-905b-45f0-b168-7dbb5e3ff1f5", "result_sha256": "b190a39c847d1aa78ef377a73122c8bec237a67a0d8d7946378e3560e045a6c5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.5690809070365503, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:21.999975+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "2a48d21a-97cf-46be-96b0-73f3d242db61", "result_sha256": "26796d091ee9cf4a6d733755c3ddb9bfa20d579a185854c3bc8902a62e182d8d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 4.1238349709892645, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:30.457790+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "79ad2fef-1849-4ab5-bdc9-556476a22b7b", "result_sha256": "b58a77449d90bb6d463548bd30a0db1770e00e7ed43d9829a8c24705122e1730", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.347845523036085, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:52.722731+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "3b7785cf-f581-4fc2-93dd-1fc9084d838a", "result_sha256": "c04ee6c14c78e10161e245b2134f3a4e007ffef06d0dcc828b561520df0e14f4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.330507531063631, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:29.221588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "90fb26a8-b013-4f76-aded-7c76d90a6d35", "result_sha256": "beca4959689a0c71f38ce927d3087fa44358cdbb076bb40a35ade89ff10bff6d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02881, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.703386669047177, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:51.251655+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "72310f8c-5899-43ae-a648-6e44d7bb13a4", "result_sha256": "742d94dcb4e6be90ef17a941e502ceb4b30525788b866101ed238cbbb2909d32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 5.196818795055151, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:58.167253+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "51078d91-a5a4-4d4c-bf69-b20023f4692b", "result_sha256": "6b38cb58b0da33f9690c28682e29c87531de1719fed146c33343efe334e86b42", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.209222075995058, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:00.850620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "570d34ad-ac7d-41f8-8d3c-e11c08ed3e9c", "result_sha256": "6740eee65870403fd62777528a0b5cae204b65940a99fac1258420fd6f0ae149", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.323817390948534, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:01.381647+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "8df445bf-ba7c-45fc-bcac-dc16a7ac1043", "result_sha256": "3cf411fecde35010077c5f489462e1fa96bb98a69b3dfd7ff0fe68982480c819", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.398761176969856, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:06.007341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f775781-7fe5-4081-8d75-d9bbaf9111e4", "result_sha256": "8e43643674682ffd74e4767faac1f9114fc13ff094cb21f4cfc2f187899c1d4d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.041031491942704, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:05.504208+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "8b872019-5bb2-495d-87b7-440bfcd2301f", "result_sha256": "11897f6ec0e35787a80e29becb0e876c0b974bc1886e844c8efcb303a6a88378", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.040670281043276, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:22:11.304880+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "974d655b-8d48-444c-8c8f-4655b443b630", "result_sha256": "78141a5114cc37b141014182d2b369d7f8f7bd10022edd23b78de6d74c524e61", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.166149959084578, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:18:42.428856+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c75ee848-67e2-4a0c-bb6d-0ae5a2db78c4", "result_sha256": "36c1df269e3264c2d62d58ad97b156a306acadd01d9b89b36cca35d218e19622", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.27214603300672, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:42.986003+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "093d2600-a568-443a-95eb-da8e8d73e335", "result_sha256": "d5a273d4ae1c2c0c516546fae92a584847a2ef6efc7653cb857689f326d3444e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.404964267043397, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:10:03.394417+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "194e3f37-d0b0-49c3-99ab-8945e67dd954", "result_sha256": "d71bc1730a6ad43252fc68033471b4c385dbcad7869bb457a9d66c3c142af23a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.5349948490038514, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:22.411768+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "26227cbe-a182-4903-9695-0c7964072c3b", "result_sha256": "649833888a4d26bbd1312a81ffd93a450311a5f33919a5ddc737e1880765fc6f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.007874668925069, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:08.116881+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5bfe802-36a9-43d7-a101-0741bc2b98d2", "result_sha256": "cdda9272ab8ab9a63b37be1da11cb7f2ace2763dcfd6968d10e094657c41e20e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.167705429950729, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:51.535491+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "219c776b-22b5-40dd-8521-444ae766670e", "result_sha256": "29deea02b2c8539edd852999865fa97f01f0097c2321678536b102617752de73", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 4.511069120024331, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:48.475940+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "89d05acf-7179-4b2b-8ab9-e9f152a0e95b", "result_sha256": "cf4743991b1f40149c76af3553286c54d5f9d65db0673350dc1d7139d888ba37", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.644655196927488, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:02.534502+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "911176ea-2c29-498d-81ab-fba1c897c047", "result_sha256": "a7ace3ac963f95bfd17d928c535559d7a368ca8aff4913eb86c71f6001c51043", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 5.931666451971978, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:15:00.040375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b2ff857b-5285-417c-95be-da95c67af655", "result_sha256": "b230f04120174e318cd4e7c66464fcb43a6bf2426644dac6f9cc6e17968cfe4b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2952, "key_errors": null, "latency_sec": 3.9703814279055223, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:44.173694+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "057a6b5c-0a32-440d-a9f1-f9af3e056896", "result_sha256": "606dbbad414b101b55533a7e08826cae074076a439a8eab25b06210558f6d91f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 3.2990403630537912, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:39.892340+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a750809-70d3-493f-85db-5ff79f024057", "result_sha256": "3d15085e99219826ff4d71fe634ef8e3db8dc73af59172d9ad60a880a041376c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 4.364495599991642, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:11.188417+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "77e0755b-1346-420f-bc18-bb00017be293", "result_sha256": "9ca09942b929ef268f695eae1c57ea55e3260c04f7bfd510de8e527902277820", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 2.7706054889131337, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:27.674730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "629bb7e2-df7e-4459-86e4-49189898928c", "result_sha256": "62e27157c55bd1e563ea86142ae6c3470afc8e7a35ea0d1fc1ee75f39093d4a4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 4.386352004017681, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:24.588984+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8d286c8-2f2b-4f65-b68f-20df77f3f4ff", "result_sha256": "2fb33358a45ef0966d974079c1460c28d6d2df6271e00681387792996123b2fd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 3.5332975139608607, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:33.657755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "0528fa34-08ce-452d-86fa-a36907d40ba4", "result_sha256": "48ca4a73fc6810eca18fbaa52e3c13d2db96ebdd922bc751a809f67860e49b87", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 3.8165724920108914, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:00.586849+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c96215a-4ffe-4ee1-be01-18a703016d77", "result_sha256": "836d12fd72c5ce8eba94fa611ad5ac636fb4305747affa89d7493071d52088ea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 3.533167466055602, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:41.555070+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b44deff2-275b-4fd9-b6a3-a3903c21c900", "result_sha256": "93f2f7a032235b3eb8f10af6f1fc21faaa1727e4b00745cf3173478abe2a6c0f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02844, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2789, "key_errors": null, "latency_sec": 3.225401822011918, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:15.342070+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "273d83d7-5e3e-4421-a5ad-00f1b76429bd", "result_sha256": "b09645277b180ab43dffbe4c5e47b7be5a594d5a06daba7ed00e1a5a4bcb5680", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 5.548872505896725, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:08:58.376790+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ad90da7-3ebe-4978-ba79-113665ee3b7e", "result_sha256": "e8b9e085421325a131ce79425cfc8d53eb2f484180d6d5a9e3d2c2e8df6d2cb6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.681650090031326, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T12:09:37.369844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4757a8f8-95fd-4fc1-97f2-3cc428b991d7", "result_sha256": "2997cdd8fce30f30b86f6fd35f7c3f4c3b97e1051649c56937222a91562da98b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.7645284979371354, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:56.501075+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d468253-bb8c-4549-8050-1825de939bbb", "result_sha256": "39655d8f680ab8a5d3e2dd952a8e99343e78038aa0aff315d3c0a3afc7828c69", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.922030708985403, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:13:56.560618+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2503402-9df7-4896-83ff-c14b880779bf", "result_sha256": "875a78a4a7c2c0cdc5ede9f88d4f4248143c34feb7b1244b776d0505f806af09", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.0932785290060565, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:12:15.349526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "5115d4c4-e0db-41ce-996c-2d5a78560b0d", "result_sha256": "290fe731bd64382d84db4ac74f7c1c6e44c399131cb5c0c8147b0aea3d527a51", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.059214233071543, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:11:31.369951+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "6f3b741b-d367-4421-93ef-b8fd4bb15bba", "result_sha256": "ec2735d1d415101f202bef4c97cfdf4e63d88bed2f5144ac53cb3479fd77fd82", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.930992565001361, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:25.803229+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "06a49705-5428-4589-9b05-2a84a865dd30", "result_sha256": "bcd7bb30240d4fc46a9d29f6a991ce72a223160ed02c3e735d53f16b435a7745", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.553407175000757, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:16.143045+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b2fc2f8d-dc50-4d6f-b810-bf000ce30e7e", "result_sha256": "e5f458e55c491eb5f81407e016bfa2d5cf2c904e8e864e2dcf4ab96b57d42d84", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.8218922490486875, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:22.467204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c1ee6aa-9836-4cd3-b062-95c94835aeab", "result_sha256": "5b67f46b288f749b62eb259b7376ad931ac7550b743e2d45b5f2cef80f044b43", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.7571636049542576, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:15:21.255457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c10632ea-4cd8-4f9c-9d82-5c4fa3dbbb35", "result_sha256": "d499b5bcc15a55d5a8ff21f8ee94b69fb558e96b0b0d77d7403688ff8437bd45", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.5814688249956816, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:50.025365+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "029c40c7-106d-4ddf-93df-3589a3e97c9a", "result_sha256": "d514ced44981ef552595ce3f6bc7043fd5853c4a740b135f87a907ee2ed27237", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 5.972971174051054, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T12:09:23.672713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "8de28c8e-8b48-47dc-95f2-1471594c7228", "result_sha256": "c0799dce665d97ed783b5be7fae5c694d55c37ac3d5f866c1d26d66600431f7d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.2592387000331655, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T12:09:44.961639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "cab827fe-ff32-4cf9-b535-eb913399177c", "result_sha256": "15df5969aacde53678ba8e685a1e3d3fdf00602591d6577055e240a92b59e9dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 2.552975346916355, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:42.591417+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b29aa5e-658b-4bd7-b1f0-dfe34d2ffece", "result_sha256": "447272826e8997b55a83da770f0d7ace6030a27621e39fa8e9a6c222d2463dbf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.494633733993396, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:39.741190+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d88facd-23ca-4c44-92bb-8fcc002cc3a0", "result_sha256": "65a48152416f55d76c39aa2a839750fd1eccf57cd40c747166cfc6e2c770b033", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.028682340984233, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:33.934225+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "6605725e-a96f-4675-83fb-40c7f3167463", "result_sha256": "1072b618d6687f849259cd8655d6599a40752ec91830f53cc8b600c9dc57e897", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03041, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 5.444923617062159, "model_name": "claude-fable-5-1", "output_tokens": 21, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:23:41.817937+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "fde3f2b7-b75e-4bec-b3ee-956268cd2568", "result_sha256": "8ecb4b5513ac450cb6af9ccfe0c6867804490aaefbadda104c6f1d4469a0e06a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03021, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.916295870905742, "model_name": "claude-fable-5-1", "output_tokens": 17, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:25.576978+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "45a53255-7902-4839-9193-ab8431303e99", "result_sha256": "cb819dc6c493e2d762f618421cabe40675207bb9afb835462f9becf11c031ade", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03056, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 4.9368084309389815, "model_name": "claude-fable-5-1", "output_tokens": 24, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:17:04.027108+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "ef5ae975-e187-4a5c-9cfe-9b40df9223fd", "result_sha256": "7d70b7930d288554180849ff8d8560c6c9eebc6175cfc1900d85811581bd5cc8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03016, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 3.9578621711116284, "model_name": "claude-fable-5-1", "output_tokens": 16, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:21:02.652018+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "5dbe23d4-6411-4c91-88aa-d8b20b6690ca", "result_sha256": "001a6a505cc9b9bc789ef1031124711c3e5bc6affcd3798c60dca7f14b1b576d", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.03061, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 5.481913976022042, "model_name": "claude-fable-5-1", "output_tokens": 25, "prediction": { "choice": "D" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:01.779749+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff31a3b3-e853-43ff-965f-9ef19d877fd5", "result_sha256": "d29cdf3a40b1575475e4595aa091be8d2163210074d735d125be6b2006ad0f82", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02991, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 6.844648545025848, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:10.512740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "9fc02156-0325-4a4b-9ce4-a7dba938590c", "result_sha256": "ac3369a65d6982e3e05853ee394cc12f4e5c86fb5c76ebb99f4c966ebc1ac05c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.03001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 6.725144563009962, "model_name": "claude-fable-5-1", "output_tokens": 13, "prediction": { "choice": "B" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:36.683708+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ad713a8-47a6-4483-a1bf-58a25c33bd17", "result_sha256": "639e6d862218841c970f755bfe1cbd01ea2b15403b0fe8d5834ea6fb436a1808", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.03021, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2936, "key_errors": null, "latency_sec": 5.382723885006271, "model_name": "claude-fable-5-1", "output_tokens": 17, "prediction": { "choice": "A" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:55.583082+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8aa5654-c651-41e4-9b10-c8de1b968d00", "result_sha256": "005dca334ddf85dc6acafbdcf4521d95e4f2a797350d8e3c5eece698eea78fe1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.276497737970203, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:07.453977+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a69b8f2b-b590-4add-94a2-73373d44366b", "result_sha256": "c869aeb4fab2551445d59b049c095abfa9a145ea2da0da7aeaf180cc4087cc87", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 4.0602688289945945, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:58.866577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4c82927-8f02-40e3-bfb7-afe4c17c66a4", "result_sha256": "51579b98d122c023830e9777493e59b8135837a2311d51e0c8627b99bab05aab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.680386846070178, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:11.618838+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "48c3b5e7-1bab-43f2-b91e-05ca81c26a36", "result_sha256": "b37700cf286e7f3713347ae5de09072f6888fe306c09615bc2c4262e79c04a6b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.553944182000123, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:52.352706+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "434c7e36-6fee-49ed-afe7-808251ca3ccd", "result_sha256": "32eedef995f41a06bf81d8f66fbe5536278623bc8f959b86f8c366bf25c4e3de", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.169032997917384, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:58.823933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "3dc58cca-33df-4017-bc2c-e9c5b748d28f", "result_sha256": "e3819abd671eda8ab0fa76244f1cf8aec9dec854875131acdfc0aec43681369e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 4.4776264450047165, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:00.610077+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "9fead7dd-6a1b-4e3b-af4c-8a4acef1bff1", "result_sha256": "6443eb24efaf74a6b7e42794e245e407d64262ea750252c23c2fedc43cba7dd0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.5113138390006498, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:18.377474+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "09e1123f-bb09-461d-ac82-de4fd239698a", "result_sha256": "3c3bc7653785f507d25aee69940138dec1ca7440b2cfd82abd9ae34dc1070d99", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 7.876769835012965, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:40.681877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "67ffadb9-57c9-428d-a168-cbe3ab302374", "result_sha256": "871bb615366f408acd650e974d989e3c4fecd818e9453170654ed279b81fad52", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 5.115354005014524, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:08.652953+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "afbcf0fe-e9d1-4736-8aca-eb0c3080b8b3", "result_sha256": "87652fb6bd5cd572c8bcd8e3021295b9141aacfc039f619560e649cfecd30f8f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.930145274964161, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:33.162311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "abce13e1-3ef4-49d8-b68d-c3b976fcea7c", "result_sha256": "e34a86398a48e27591d98ab35271d4adb4f9b70c3b59b1b789ca9e2c0dddca5e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 4.054514944902621, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:18.469118+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a002508a-6e4c-4707-96c0-e3273487ef6f", "result_sha256": "050609c7a12681b7c50a9fe6cd56f6dd36ab296ac6aa55c92fcba0d3ad5e57d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02866, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2811, "key_errors": null, "latency_sec": 3.579432393074967, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:49.667721+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "f15cc972-deaf-4c24-8adc-ffa4c2efeee1", "result_sha256": "b611268b8e0baf029e5452653b78e309080554990957c05cacf36fe97eb0adf3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 11.1389847319806, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:17.369161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "00293b83-7712-4e5a-81a0-57f208a9aa61", "result_sha256": "f0035c83b725f7d0eda2600a10f23c6915f1bbf67b0fd66f4106fc5b9d8118d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.132166153984144, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:23.765329+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "0856042b-eb44-48c9-9866-06743dd39fc0", "result_sha256": "8eda48fc810e45491c8bec062ffc530a5596289d6629284e248c3693ed8f5fb1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.9459432050352916, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:41.013191+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "9b43d4f0-9402-4ac2-8783-8763339b2e10", "result_sha256": "251d6a57fb076b9c1815ca5984add54d8906d14c2b86ada5e1a5d8622e55998a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 2.851811353932135, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:41.207215+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c8d01e0c-1dbd-40d5-adec-30ced8c4ea56", "result_sha256": "b74b95761c226ad4de5666bf1b96f9513d47bea5f3863bd6ca4bc7de6ad37388", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.366358311031945, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:46.409594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "79ab5cac-a9dc-4bd5-adbe-f8ae2296d9b8", "result_sha256": "9a34d1b66c5bdcad89db12cbbe3df25df223aff0a26eb058cb9f02fc7427b117", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.222509662969969, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:42.937778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a80381df-d4c8-447e-97db-d25bc9360fcb", "result_sha256": "4c8ad067fe2a65e25643ef327d8f25878d318e5483b4a86bb264cf58a56969a2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.055100120021962, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:55.843441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "e850aa5f-3c51-433a-94f2-420d68783f8c", "result_sha256": "cb56c395db9b1e9c970b2782d215207aae8842898c8368963e5d0286a6cb9a2f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.497054898063652, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:59.404242+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "61c9ee67-cf11-4341-906a-9e55ed09ad62", "result_sha256": "5492588b7f526801c5b303d1aec12252b266b868c9f841ccc168e2164ef5ff7f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 6.0935818259604275, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:40.020525+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "bc82d923-41f3-49e3-acde-7db34880f6bd", "result_sha256": "cdb568fbc6440630e3d1d08eb5589295f1d714cec87a1aa6137b96b905aaad32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.6370941780041903, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:48.005395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ff5f35a-caa0-4224-809d-442781370eec", "result_sha256": "250762723e7f1b9453b82fbd39f2ee9112d90e7efe3b60186d5fd499058ab1ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.2259565429994836, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:32.196661+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "5bb9dea1-9cdb-4087-bce8-fcb08cdb4bdb", "result_sha256": "34460a7c84d05ca54fe982b88dd452c9295f2d98e9e0adc29821ba0264f6e404", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.030306814005598, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:09.537482+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "5058af52-cc32-454b-bf1d-7cb060b32645", "result_sha256": "3fb34a59c9bc041d9d4048fba40f4bdcfc870244483eed1f8b1625ba2feb0138", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02918, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.716419636970386, "model_name": "claude-fable-5-1", "output_tokens": 18, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:04.898775+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "bba6eaa8-7637-45cc-9f33-2b9fd8017f13", "result_sha256": "646309b456872e32f59268494ac166e1cef8ec0e50237ef341eed4d389dc11fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02983, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.743830605992116, "model_name": "claude-fable-5-1", "output_tokens": 31, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": " {\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:14.630882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "14009b0c-c0af-4d87-8c6d-c88b3b03948a", "result_sha256": "ad23b5bbf3265ba16ecc0e364a711eb38f76e0512478806f656a6f428d2df195", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02963, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 7.111663406016305, "model_name": "claude-fable-5-1", "output_tokens": 27, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:16.163490+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "577a75e5-8798-430a-ad43-8e3e0f1f9f82", "result_sha256": "87365f5fdaaccdc6d0234946e86a334fa15e5ab3955c22321c940f377e101000", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02963, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.540425140992738, "model_name": "claude-fable-5-1", "output_tokens": 27, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:30.125283+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "8b4298fe-1883-4ad5-8e01-ab51e93ef260", "result_sha256": "127b02544bbc7abccb871db1bbd6cdde53e20acb70b3405a4607e7c364236064", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02928, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.721293470007367, "model_name": "claude-fable-5-1", "output_tokens": 20, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:28.265668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "12a67fbc-5ca4-4097-8d90-1c61482bb58e", "result_sha256": "ae00689f30cdef5d05cc359fc0bccf2f6ec32f8e2786ed67a2c23ec0db16c60c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02963, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.700544994091615, "model_name": "claude-fable-5-1", "output_tokens": 27, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:40.972501+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "29bb9397-5fbb-4880-a080-9ad673060205", "result_sha256": "0a2dcf3cf05486bfac10dc27d670558a350b637dd42ea21e6dd2a83e0f3002f8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02963, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.677318845991977, "model_name": "claude-fable-5-1", "output_tokens": 27, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": " {\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:59.248105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "40e26731-ab32-401e-87d4-d958a67c4785", "result_sha256": "39fc0f4fabdb7603a0850203484dabf00d64b76b757d8f18869f4e576b405b84", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02903, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.1393680709879845, "model_name": "claude-fable-5-1", "output_tokens": 15, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:27.121715+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "69691359-7df5-418e-ac93-d6fccac4694e", "result_sha256": "663304fe3a70b1e2c41dcfbf11314cf1f59b01a120220d2ed75a71b1fa74b702", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02973, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.628084594965912, "model_name": "claude-fable-5-1", "output_tokens": 29, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:39.193995+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "dd5df510-f6ea-417b-99e4-68c1d8e278ad", "result_sha256": "8c5a5772ff4624e7cb6bfe91c3181dd306b49f1b20876ec687343af64dc2ef62", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02903, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.157988767954521, "model_name": "claude-fable-5-1", "output_tokens": 15, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:17.082605+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4c51741-d800-44b8-8397-96caed8b24fc", "result_sha256": "45e97d867f1043a54bf2aa66ac69baa46f04956c048279afaec27eec88c387bf", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02963, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.374888858990744, "model_name": "claude-fable-5-1", "output_tokens": 27, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:08.871596+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "13bd8d04-efdc-46f8-b7a1-d0c3439b7385", "result_sha256": "ce851d2e29c414f5cb803601a57fbf769d189562c51266dabeb96650c4f0647c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02913, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.021841080044396, "model_name": "claude-fable-5-1", "output_tokens": 17, "prediction": { "choice": "B" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:21.360192+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "2e415666-94dd-4290-8e03-ca91728e3740", "result_sha256": "e1d3d775af44bdbd4f19b28f1666e0c7a55b962ab7722da838112f646d54ecc8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.227574602002278, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:03.582004+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "50ca9bad-6a90-43c6-a19f-aace77dc349d", "result_sha256": "1d77069b68c36b5ab357395923b207f2b3293ebf4ff164931034a95c5ba2917e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.060978532070294, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:10.963936+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "3727a07a-c9c9-4234-a257-cf4ba66adb1d", "result_sha256": "b25351e20601a85308fba043fcc61c2cc72347116272dead551bff511a14d6da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 5.609500819933601, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:04.363542+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f29e88b-1101-40a7-ba77-40d7dcd95a8e", "result_sha256": "e955b617e61a8176414abb7fad7dd9b1f464749a866516f067a71ef7713adb72", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.586402195971459, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:52.702692+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "de3cd02c-b9d1-4780-938e-e28bd0cb7171", "result_sha256": "c7bb3d4dc1e602cd752128ec79a7957972e2ffbfe8819cbb8053dcd04f90ccb2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.507002404076047, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:32.112471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c6c2cbdb-1286-44e4-8325-2f57d9393001", "result_sha256": "0a64a6cc763248547d4732d47c4faf4ab2c56ca773f7f7253f98448a9f81e01a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.5257802330888808, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:44.354436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "43dc4b09-9a92-4304-a278-ab972ea1fbd6", "result_sha256": "11a00380ab4ae29340860fb6c1645df4d786da27db2c0c0ac7cb3dd5eae75ef9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.2908012910047546, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:50.551585+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4987e724-8150-449a-ac08-336caa82c0a0", "result_sha256": "34a59335d85a1510c55ed0c68fccbee8574e273958c2902a5558750ebefa06f5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.177354063023813, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:03.642864+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "25c3515c-7674-417a-803b-f7f015338ae7", "result_sha256": "a86a1c001eec927fe5c261acf3861d2aea22ae595323e6f6edc3204c2d017515", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.895375888911076, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:30.724853+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c60cef45-e089-4283-8bd1-ff901c4e4b90", "result_sha256": "a49bf32285747a0bcbfe9e6b1f3420bf073f6da9015169ab88948fcc76498131", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 4.61533987603616, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:27.260873+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "68ce1d65-1184-4556-b4d5-b888f4f6c66c", "result_sha256": "e5e22fdd62002f2ded2c85f21538e455b604d94d5e9993f3396c4dcd9b76665f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.385378777049482, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:52.887843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b39fa5db-5074-4e2c-9c12-2cc7a2d7679b", "result_sha256": "5d7dc158a8222f7dda87b42b353bd24520f4eff53eb108a331bd3efde8cc73ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02843, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2788, "key_errors": null, "latency_sec": 3.8657017110381275, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:51.634852+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c48d886-a31d-4e20-957f-c473190f2cc8", "result_sha256": "54466ac9217d9553ed83e41db6c38e0d4807c4b0db3bf9e3140364ba6ffce7ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.479193859035149, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:55.045720+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "7ae4e905-01d1-4de8-8584-e7678fc4c133", "result_sha256": "3c69c207b470fcf6ca32da2ebd1dce40d0ac43879210d6adfe75255fd8daf208", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 4.834709201939404, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:10.958188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "99f77b4a-237b-433c-8024-d74d87160b07", "result_sha256": "a112fe5185d6083a286bc0e5581e3900f8af2a8c8909c3371d410ff5bac037f7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 4.312621732009575, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:45.258527+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "6a003c93-81a9-4cfa-9a7e-b74c784b60c7", "result_sha256": "f3f17e6fa0c80dd3153353eacbc85b65e92371d948509ec62a2a2e5870821945", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.416067873011343, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:52.514717+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "2cc42c2d-0e57-4984-a6a7-c058fd084607", "result_sha256": "d655a19e0d62223abcbb4ef96e17a493c6d80948c1a3418d6955f06f365c520c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 7.499128506984562, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:30.104896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4ebc6650-c67d-442e-bbfc-d03353fd85f5", "result_sha256": "de38cac0cb3d91d57f2731e554ddb799ece149531028fde5d619aecf3d14525a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 6.0084520620293915, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:20.942510+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b9351743-e51e-4166-87b0-aac9e36fda25", "result_sha256": "fd17de605d0304f1a24f560ab6cd8bf54aaad614b6d90e6ea643fbe934e40d3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 4.257706481963396, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:51.927479+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "f532e358-be3c-4127-89d3-9915cc8b14da", "result_sha256": "90597161a22f3f5bf3b8f558a48dd16798e90854ed7d75867fb4de8c705cf313", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.2357888789847493, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:40.038049+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "6fddecce-fe2c-4782-ae0b-04b263bb1e04", "result_sha256": "524153fbbd3a26a6c455c7e3be58d8a84191416329859035652a9f5c54a6d38e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.326130390050821, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:56.290617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "21aaaeaa-1c30-4013-8bc5-e04970b86109", "result_sha256": "3f347a5f37ea96afb16b244ce9c2451a37efc0720367eecd6bc59728670cb0ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.4309307100484148, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:04.797927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b1967f1-495f-4224-9f73-d22b95271790", "result_sha256": "8121fcd2a88922f79860f2cf3740720aacc29d972697b11100cc39754b014e77", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.861141834058799, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:27.194003+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "03306168-3844-4fba-b0d8-d3d69ae2585c", "result_sha256": "febc245759e002f9f69c3b8e5505f6e5c270ad670b1e9aa8b483463b41fa2560", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0287, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2815, "key_errors": null, "latency_sec": 3.4893715570215136, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:15.651933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "0cce1588-0d63-4643-b2d6-003aa934b452", "result_sha256": "dd5679c7af050e30b96e3ebbf20508be0e787e1d0a5582c24424ff4e9e9946cd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.8932333999546245, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:42.966797+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "43c0e1aa-c7b6-4d49-9046-4e0ea22e0fd6", "result_sha256": "6381aa608accac727d40eade1eb663c924f25827715ea5cdef8672eb1f859188", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 6.21435724501498, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:17.812363+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "5c53d288-2496-4555-a29d-86cf56d796a2", "result_sha256": "0ae2dba174ed25047c1d4f57df7423b7dceb286f90696e5e168d142a5f026ce4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.352794138016179, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:28.829253+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "8db1f38b-4029-4d3b-9519-c72bbbaef44b", "result_sha256": "c5dd6558d05f4a605cae75e4a25614436b3003bac2a76ee73461f30114a33d5c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02875, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 4.505828456021845, "model_name": "claude-fable-5-1", "output_tokens": 13, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:10.403674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5e42a50-2354-4054-993e-db196c167107", "result_sha256": "51fe91433e854bd85a8620b3b24de8b3ffa94f8d216cfe9a6e1d8ee0f4959533", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.274813943076879, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:06.336773+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "65606005-6816-4083-b185-0c563037376f", "result_sha256": "bde5adf477742ff089323159242d7145ca0b727499ae4e0aa0744e5545e0bdcf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 4.631953828036785, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:46.314032+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "df941a97-5ad5-455b-8fff-38b030be13ad", "result_sha256": "167caab10975ae3a5c5d66481f72f72915bf255e2e81c7f3a23b48d54c0a032d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 5.104505092953332, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:18.246353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "eefc0821-1350-4c9f-b8d3-b5f4af1cf381", "result_sha256": "982bee21755fcc0c1f1110a42b35beb32828cf212dc32e48b8def374305cb18a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.346115943044424, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:29.720760+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "be0b6439-e86b-470d-acb7-efb42be223b3", "result_sha256": "3aec41db2f3a2f1f6cc5879a57319f8d818a67d1f33c0fc7e59d79b1ed527972", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.9298677350161597, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:44.449628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8ab158a-2169-48bd-9113-de0842505956", "result_sha256": "2c6c4109a409bb109a6e3c6c8aa471fb83ca46a966e9f097ac9ce56b416cbe7f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.345005262992345, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:22.067943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "58f5bd4c-1f00-408a-97b6-44fb99be58d9", "result_sha256": "f0eb812ae11ce7aa8703366def672b47a5f9d16ba20f93bb085f8f96d2e1868f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.34217372094281, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:45.951262+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "755af591-aad7-41fe-8f18-1cca2330f2e1", "result_sha256": "3ef233c71734f22e7503c0d93abe1fcebec7b8b73ddcbbb0761a0b9a6fd2e147", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02865, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2810, "key_errors": null, "latency_sec": 3.4513665140839294, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:42.645743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d44136a8-7a4c-4682-8119-25c7c4497e35", "result_sha256": "2abb519e92c54334f721d9e501dfc17e1c2e31e69aefa3cea03da103431f5b25", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 3.9468813539715484, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:10:01.580665+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d1b0826a-568f-4533-933c-6311bc15610d", "result_sha256": "165fce69c67624746aaf0a8a1ca3166e320a7da5c142acc51a8f5f6fe48acf29", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 7.033768536988646, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:29.039854+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "8fb2ff0d-faf9-4eb6-8810-bc429976db3e", "result_sha256": "1195e9ad5642ab5e501bd01964c49c3a44622843dde34d485fad36dde958c68a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 3.9401558820391074, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:22.766208+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "157285eb-cd0d-4442-9a91-3367486862c6", "result_sha256": "5c16fe562529247eba04097e05c730c7a0278aad6bb3caaa996ba3c3de3c38b0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 3.568294532946311, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:38.494377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "46bcf131-4dec-40a5-9a57-12bbe2df197a", "result_sha256": "c5c9263f56f51d89cff645f5b854d9b97855edd05d4b7e227599dbfe859fa363", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03866, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 5.484394251019694, "model_name": "claude-fable-5-1", "output_tokens": 204, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:48.535350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "0931c4dc-51c0-4de6-8a97-d100d79f05e1", "result_sha256": "05255abdc916a449d130549a03ccd8e240d5e94eb1e1c39c34f7913b0754a7c9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03881, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 6.554766725981608, "model_name": "claude-fable-5-1", "output_tokens": 207, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:12.745762+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "14571904-0317-41a3-8b38-0b7a62174760", "result_sha256": "e71bbd966520a4d520cf12d8a930393d4b8eecd5e4d1e7d466c24eece1f2fce9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03711, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 5.84420070098713, "model_name": "claude-fable-5-1", "output_tokens": 173, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:46.237830+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "dc6aec37-c5d1-4004-a3cd-b6df1123d5ee", "result_sha256": "f8e23c199fbfbb36b82480df3b23af38b374e1f0e4afa02b04f90a4de88da539", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03931, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 5.922074805013835, "model_name": "claude-fable-5-1", "output_tokens": 217, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:55.115370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "465e46d2-9388-48d4-a697-ed3dbeeeb72a", "result_sha256": "808374d3fa2fe03413a445f58f996790c8956b73d4a30fd3bbc2ca965b354e49", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 3.815051767975092, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:38.517985+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "f07431e5-680d-4d40-b24d-6edaca5c1f92", "result_sha256": "d8e5a9e643a7e14b73b277afc85e581dc0546c1200784792885a95e43cf6e3c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 5.780746399075724, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:43.433246+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "11d862c6-5dfb-4c15-820b-95cc03a79dd3", "result_sha256": "5a477e56328612d07b360746136f629d4d9fb997393d335a05b25a75329469ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 2.9290942930383608, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:16.010287+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5559680-7ddb-449a-a76a-983a962bd261", "result_sha256": "fec0abee803cbc2d6bdd8845165a14f150272c3175a0f16973739fdb0d6896c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02901, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2846, "key_errors": null, "latency_sec": 4.640244425972924, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:34.822499+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "88ed5c92-bb7a-4d43-977f-c50b3fb89e21", "result_sha256": "ebd9ceb4203831c895682e7589baf17215703e63ebf12be8d572413450a2f0a3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.577894364017993, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:48.789291+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "26c1b3d8-ba72-4412-ab94-ba1d61a4a604", "result_sha256": "2a61d86bdd000edce30e561751501288555462cc15b2cc18a6724e99ea5426dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.3536798630375415, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:03.029455+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7cd100c-41ed-415a-bdd5-837528ae61bd", "result_sha256": "2aab0a73edc875e588e0ce0098cdce55906c050bab66a47b2c809546749ac2bb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.5249353679828346, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:22.584448+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "f96bf23d-353b-4ecb-97ae-a2a4958fe9e4", "result_sha256": "8992307d845732ddcc8a5b976860a8b6b291baa1b055fcf0a146b2c63f0f51a1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.466801682021469, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:55.647327+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c83f2d0-c052-44c2-aa64-311630deb294", "result_sha256": "50f5e5e8f77577358923fb5802e2dff5bcccdc21f7730d44275665bea61f6a2c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.303683356032707, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:48.088330+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "ba8a9362-6816-4342-b396-d83c34b0a6ff", "result_sha256": "33774410e9fd9808ed19669682c9c22cebbb7e0aae243702ff84a768af1d767c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 2.5410386759322137, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:19.930383+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c4c97121-fd43-4914-a080-3c692ef690a7", "result_sha256": "ee46a6a77f06a9b08a6652785007e44daf1f7f97c23a87141f652ee5e1f5bc17", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.807485605007969, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:27.749664+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "69a76771-101b-4442-b5c6-728da82f995f", "result_sha256": "7dab1a0258d8f36b0eb0ba5a3cc0214559e01ac661a9e72a7ace5f7163c7b47a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.255968836019747, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:45.903416+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "7dc47d8b-f65b-4653-8e4b-b332a00f59c5", "result_sha256": "e9d824c24d8cb82f116fdf46cb4a93e6c4eac3a4657bd9239736a338d040a55c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.791079943999648, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:09.970811+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "732750be-6292-4a43-9f9a-c2b9a11950f1", "result_sha256": "56c1d090c5cf17f053ef319848141017a185e566faf0c8dac392ef607bb7bb1c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 5.5426894560223445, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:13.268494+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "40cf3280-daa1-4d76-8c30-4a121340835a", "result_sha256": "6ab8828e6ced4ebedd1450c11f2ed8ad449d2a6ada35a18e98fb2491772825f5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.8611769980052486, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:34.408194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "72fc1425-d064-4fab-999f-aa3d907772e3", "result_sha256": "e7e3bea31d3eb314b7e5816d04c553ab0357504a261318da32390f7901e134c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.1571320169605315, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:56.146978+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "63b490e2-8a4b-4f88-a0da-80819b281b0b", "result_sha256": "f00f7ab10e4933bc88a8bd3302f0c6d941775b4c5a32bd6be26405f9bcc080e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.660805229912512, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:18.053809+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc053485-b09d-44bf-bfba-915aa9e5a39d", "result_sha256": "a45e0354de9281df5a303467f1074b0edc3d72d257744d9b0f7c141c54620ec9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.489006040035747, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:32.639693+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d86f6f79-aab2-4d4d-91c5-f5dc272ea0f7", "result_sha256": "986ad64ed22e4b358f8cae5dab726a16f7f94729d140cc188b9f1fa3e22bc606", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 10.603448732057586, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:46.689445+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "56e8a471-e982-4a5a-bb74-9e26cc83bb70", "result_sha256": "294a319e148a7462ad3175d8dda0922e6e10c4d8ceca40e736c6c061eeb40cad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 4.020814697956666, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:33.582774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "f3c9618c-0041-4004-ab63-67ca351df30c", "result_sha256": "9304670823ebaef818eeb462630deb7658e536ec3a22d6c0bc8baf0ac4b2bed0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.859576813993044, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:13.113222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "00614acb-92cd-434e-9e15-d9efe76f08a8", "result_sha256": "4e3aedcf7d833bfdabaef77e2fdad7e95b45b40c0658c88fce3b548d7e89839e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.552591860992834, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:30.010170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "64dc6f92-f7fc-412b-bd7a-0c8027a9aa86", "result_sha256": "b4a4f5651bd86c400bd3a4b8f63c34061fa38d6a928322faa3bcba6ff5b3142d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.640097302966751, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:54.681847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "209f4188-1c25-471e-a5f5-1849dbd5def3", "result_sha256": "4632d526b7408d80f7db82bb88d421e472bd86b7419cad253f84521cf44ebb3f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.7396499280584976, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:47.810101+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c92d183e-60a3-40ec-863f-474188551cf9", "result_sha256": "7d13ebc3eb02c8fefa7165ddef5241f2eac7dc17bb8cf4081ee945dc70fabaac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.4880438189720735, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:28.808161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "579777dc-030e-49ce-8af2-4bd366427614", "result_sha256": "51115910355b549a0f2e2ac12ba082b7a8c1b6fe03ac1215f21a4a64c78f794e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.5377004919573665, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:05.817996+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "7da39f28-229f-4581-abdd-5f8abfceaa93", "result_sha256": "9d3015c205f004ece4f704ae9dd5b6d624e360349acfe6ed33f671e880522056", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.6513717600610107, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:38.960741+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "2bfc60ee-970f-4c1a-8c5f-05b4c730174e", "result_sha256": "0365f14d93b5b8b781e8621cd35cd654afe8e3992aed90570f9f50b6f2587186", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02883, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2828, "key_errors": null, "latency_sec": 3.5396450450643897, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:07.380564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "977dc55f-081f-406d-87d8-af04ca599343", "result_sha256": "e8b8b4053f26000b78342dab246659e9b8015917d24f65e6d3ba7219e95fe800", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 4.351046836003661, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:23.074725+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "32324c1b-801f-4edf-b22f-af5a352e482a", "result_sha256": "f193c908448462df8a00dd1a5e33eb99cef71839b150c8998ec9888e6ce5c0c5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.670571875059977, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:36.487999+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b21e44a4-3350-4a99-959d-8ca0dbab04a8", "result_sha256": "65c4abf3b9dab6f7b0c7671a80e90ad1c9ca94c4b8c0e9a3f2db2c83e2f71420", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.380320427007973, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:28.653357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e5cba92-680f-4ac8-8b46-40292122afcf", "result_sha256": "59f893f9d5686047ee7acd236948872b9cf2b88c3f56dfc1bc5a71de22a6414d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.319516382063739, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:20.590177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "520b1b90-2087-44b4-b7a1-f247051eda9c", "result_sha256": "7125f25839ab7fcea430cada63dc2ccc2029f8b8c53b1eb2fc5348b85a2c4d30", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.2624363750219345, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:04.130334+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "71296aa0-2c57-496f-a7bd-c3bc5ffeee6d", "result_sha256": "9a672cb7b8250944d1b3dae23fcd5ec3c0b131469195aaa81acf837ccb6d9cb9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.6302618640474975, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:04.318605+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a9fd616e-1789-4721-8a4e-0af3121ad34b", "result_sha256": "5d6afc56bbb655c90d02a7c4e64e06193f9538fa3ffb5d4ad263dea30ce44afb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 5.295351333916187, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:37.522627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "27ec30d9-2569-4644-b0ac-af991ce9adcb", "result_sha256": "261391ddadc51298d136aec2ed6c47900a73d848fa23ab5f433a770cdf0fe7e2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.53178681305144, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:24.517734+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "ca3598d2-f4e8-4efb-9665-f3a26c40649d", "result_sha256": "a78816ce0b950a632a345151951d123190ad2f18ee189c551f4a41d69733542d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.403323512058705, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:02.737044+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "97561b7a-6f6f-43c4-83c2-36676779f0b8", "result_sha256": "a2987a73c1f8068b307b2f22e2cea1b8c935315b63a5322f8e89b4ba39f61de2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 4.970326830982231, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:30.562657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "3177f5e0-b95a-4964-a804-02222a084d3a", "result_sha256": "7d6ff1b5a429bda7bbf81153d201940443154ed1d42db7cc1509d1165d39a074", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.3811056839767843, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:14.433046+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "69d11c88-2c56-432a-a970-f30630b5ed65", "result_sha256": "df47a4feadd1fe3ba13ed2b30cae234dfe0bfc0ca62227d0ac0a8f1c9f500691", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.8298568360041827, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:42.817953+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "69fb6a53-220a-4a4d-a919-5e7dfeffa329", "result_sha256": "b5d00f862f8ea7db05733d78c8a074157b0f5960ad8e0c4cc5c2a80b2fbe4c3f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.628795061027631, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:07.384502+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c6da9ade-6013-47e9-b2f5-deb6a5c4d5cf", "result_sha256": "7a4c5f8547ddf362822819e41fe180895b21dd27e46ceb4f8f237d72105a16ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.58785557304509, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:32.358131+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a32c2fa9-9528-4998-85a6-86e285aff548", "result_sha256": "075a93022df577201a9b5a29095d5df71144584f1958747e44238e97ed3b4326", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.2739437649725005, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:36.592954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c9ac72ef-1cc1-45c2-9090-9ade0eaabbea", "result_sha256": "22487b1a1b2d2c600985067cb5c451509227beffa9e31e19bae217124699f151", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.8939761959481984, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:54.390204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "8ed8e7ed-e797-4e38-8c95-485b8eb83746", "result_sha256": "84adb7a94330ec50208bc5a58305f96f6e394b575d58f9791ccde1e093f5dcac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 9.546847705962136, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:25.167516+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b97affdb-46a6-47da-a336-df9c712719ee", "result_sha256": "5bb07296e89af724d8f9d20cf82f2df31e14b480cdadca756f86f75cca127f1e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.941538208979182, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:31.722134+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "0fb8bde7-8277-4bcf-aafe-626f25878a4a", "result_sha256": "338550b121c724b7f9dbcf5a6ffd940cf831bba94333f14b13b2cd5e781afb75", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 6.711406570975669, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:13.136218+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "41da1914-bd10-49d0-9025-51a2b69037bc", "result_sha256": "00a98cc23176059c9b5f1d3e13e4dd97f3f448cc1b15771349d951ca36a7c9dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.365368698956445, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:12.143051+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c4a7797-780a-4426-a54e-3203e5eb3c2f", "result_sha256": "a90adcd198465c20ec749367a2966ed4bd99a75baf797909d5265db5535af990", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 5.814442589995451, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:48.391376+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "38408e3f-a34e-42c8-9e61-610dc0de4173", "result_sha256": "fadf4bae9fcbda22f8104c8afc8d2d5c3a851f9a5cc6e44bb0d34a9f6cdb52d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 3.49603548203595, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:58.412455+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "81e688f9-a276-4b59-a57e-d9ae9196d025", "result_sha256": "84e3d12b75350bb497539ac4f20e96fde3c37204687749365b57525e7d0956da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 12.781366653973237, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:36.793326+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8d1d325-eef1-4f1f-860e-351cfcf59fdb", "result_sha256": "e78310ce472882813544effa167c1f00533eb9c6e5a3ca45a3ed0671bad922ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02896, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2841, "key_errors": null, "latency_sec": 4.813780969008803, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:49.340661+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "1fa1145c-ba5a-49f3-92d4-cba8390e986a", "result_sha256": "721d3785b2f9ba19e22a93b8df2da23366f54f1ffdd1d9c71016c86b50ce2aea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02945, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 4.546628422103822, "model_name": "claude-fable-5-1", "output_tokens": 15, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:58.457274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "5855c4b5-b35d-4b08-89c1-944dd0f15f0e", "result_sha256": "b1bba6e878111f9ef3bf57294de7a31977024ce9e48bf41c0f1dc10536facea1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02925, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 3.379492409992963, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:18.922427+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "fb4e3c7b-179a-4253-a468-0382753fb8ca", "result_sha256": "3b22e8b98ed3dc0fcb24ae1ce777c19e3e0beca54360c82141ebb157ff199600", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0294, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 4.3097757620271295, "model_name": "claude-fable-5-1", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:12.556598+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b699219-f4b7-42b0-bf8e-d3394c52f98c", "result_sha256": "63820265b8767d27e56531097f978018d0f184c2e841058e18c71025b5df0f64", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02925, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 2.8726766069885343, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:46.355449+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "877b2d04-b3f1-44a6-94fb-30742fbc1bc5", "result_sha256": "e818411ccbb696aa69278270859be42167cadf86dae27eabdb6ab33116a865a0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02965, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 5.771281318971887, "model_name": "claude-fable-5-1", "output_tokens": 19, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:29.454354+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "96d2eb98-429f-4c87-b711-49346b351210", "result_sha256": "2a1236fa54af32e4b60890647989603897a1c0c65bb15263098827e1e4297ec5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02995, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 5.5445699649862945, "model_name": "claude-fable-5-1", "output_tokens": 25, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:10:06.312755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "59d0205c-712c-47d8-b052-92a4ecb1e9fb", "result_sha256": "5f2a74d37c9f7108e7a666eba75b12f55aa1754ce23e1f3daedfcb71a702cf04", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02975, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 5.701614364050329, "model_name": "claude-fable-5-1", "output_tokens": 21, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:16.617611+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8365643-d605-47a9-b0a2-9a2175a2571b", "result_sha256": "54d3f0ca0dccc93674a18b2034619a46d50f46079ee1433e1bc45f5dd0054aca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02955, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 4.385055043036118, "model_name": "claude-fable-5-1", "output_tokens": 17, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:44.448886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "e53f5563-8090-43d5-b098-5c37fe0c07a0", "result_sha256": "808c647d94933a45d09ee7df4bdb23cfc7439f568d5220e85e2f05a565de5360", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02935, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 4.189949057996273, "model_name": "claude-fable-5-1", "output_tokens": 13, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:38.322166+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d5f27c4-3df8-4be8-91ec-79265156e31a", "result_sha256": "e18d6b45929267ec55096788d4f582585c509a9179c56d56ac6d40bd588bc35c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02925, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 3.633980589103885, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:47.617395+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "66b07302-ab3d-4a4c-9260-b8cccade2d25", "result_sha256": "30590226d117e530cfdfd4a7eb0058817c44bd21f8479ff03c78d2f91f0d6015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02925, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 3.665623558103107, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:33.511553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "9416e1fb-e6e2-4b6e-b94d-8268cb8254f3", "result_sha256": "a4e295216999930b2e335368a96f2c9be721d42f9ef9f5c380504a0a853394a2", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0296, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2870, "key_errors": null, "latency_sec": 4.622650080942549, "model_name": "claude-fable-5-1", "output_tokens": 18, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:01.209110+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b78d1ad7-41da-44a6-828c-9b292e379766", "result_sha256": "62562efd9bb63e79c33f4d2457ae8a97647dd01f482e6e26caddc49ca8ed6dff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 4.27970632200595, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:20.728565+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "8a174329-e922-4b3d-929c-707da8d2bd8d", "result_sha256": "118ad395e530e2f5d3daf29ad3750f69fead86a30b0be86bef999918ddcaf221", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.7274722260190174, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:08:57.421267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "249c223c-1f45-4caa-8994-e78e4c3119ac", "result_sha256": "36b47b1e9b86ad03176001c7f66bf36b7c518bd2c938f4d0170e130bcb72ac07", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.40740028093569, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:45.344816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a59dd924-6ce4-47b4-a14a-8f516228825d", "result_sha256": "69ccb7da13322e7e6b4304b84437bad076474625466a47bf849c41a7f466716d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.3958913860842586, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:58.060457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "9f87a612-55e7-4860-8ca0-49a2766abd8b", "result_sha256": "164473b4cb08f2d9d0197824254f79b61d2f008af27ded25756871003312e29a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.895132091944106, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:48.922794+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "82436c08-b8f2-4e77-98fb-24c394e5750b", "result_sha256": "49684fe8a6a28037ca50b4eb7ac7dbd50373544b5e1ca85051f49fa8f1ebdb3f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.3775680540129542, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:38.821609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "e187d5b5-7613-43c3-b682-3e26028a32b4", "result_sha256": "34a2e7dcc17455f86ffd4fa50b959f1f01397378b0a51dc7323934f838def3c7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 11.16955458605662, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:20.581292+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d49b834c-9281-4d2b-b673-076c7900245f", "result_sha256": "9683f49433b8656e820c0f8060d17a71e96bfdf5c7296c5c052985643b4ec52b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 4.824583631940186, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:16.674439+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "f133073a-ae99-4146-baad-a9fdfc903d88", "result_sha256": "451159031df4b5b4890281645b23fd52e05ec29adcf952388b21185f9551737d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 5.704538463032804, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:23.523932+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "9d60b237-2f29-40e8-ad6e-e4c8a3e23bac", "result_sha256": "868d2a0eced16c6335ca96a5a53b6e00888258f32dbbe0f890a88e595dcbfab9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.5085732920560986, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:20:54.708879+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "e76d7aa3-0804-43d2-b1fc-9f3e5cce9e74", "result_sha256": "e3089f4ea89dab824a966f7bbce01aea3669406f4bf46ae18b999078afa47012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.6617531690280885, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:02.687695+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "1bc89c7b-cbe4-44f1-abb5-95b5c7fffc5d", "result_sha256": "d7ae00527a513d86610a78c961a46e8a34d2b57e38dff469d012d0928ae0ad6e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.475862516905181, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:10:03.204282+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "82290faf-4336-4512-a44f-06ce579bdea6", "result_sha256": "bd6ad0c770ee78e0716f8b8ad7da97e51e50e23ee6b79a6980a0f57afedba66e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.737586137955077, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:05.964236+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "77adc890-9021-4626-8256-28543006c1bb", "result_sha256": "a28c97a3b6f89b8e058cc1b9790d457318c55dde16407382ef87987a659c8767", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.3339415109949186, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:08.011452+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a6419779-4cb4-42a5-9448-2db831f98af6", "result_sha256": "877e0b6baacde62a380d17bdee08274ce55534c87e6dd4fe3a6439081c326c1f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 7.096835281001404, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:42.126703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "319b2749-ac3d-41ea-a0d8-9bfe69abd9f6", "result_sha256": "c4f4866c5cbf19897407fadcb1043f71b1ba11a4e5a159ca6c5a804a8b8c1666", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02907, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2852, "key_errors": null, "latency_sec": 3.4885768078966066, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:57.007009+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c175c927-3e9b-4d84-b272-35e719d5c236", "result_sha256": "b69b729eb50b6636ec0e3322c3cd2b75199065ef9968f1adb50144a55d56a129", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.6062139560235664, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:51.429265+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "18b3d565-5af5-4275-9821-52d3d161bce1", "result_sha256": "ba1e650d9ef738376cead0824d11c49539d51e713889b752505ef34669b979f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 4.572299193008803, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:31.810225+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "447c5b63-d7a3-47ca-9512-e9084c7796c3", "result_sha256": "638e285b48e42cd737789cd7411c723f141193216bd9d0401d208e71d3f39afb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 4.350167638040148, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:31.867780+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "44ff949a-6a5d-4d82-9ee1-503797cb37ac", "result_sha256": "1ce8fe3c2f8e62adf21dd7ad3a56fa3082300d1341d778ebba074933d1a3f7b1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.509569860994816, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:26.705794+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "7bf44369-9d3a-421f-a283-1f218ac8f3c1", "result_sha256": "ab4a0e8d841802ed329bf0317fbc83b7fd90b997c6969d77c7c0ed04ebffa954", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 2.739163312013261, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:46.881361+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a4dcbbe-34d8-4ad2-b67e-8281a61574a2", "result_sha256": "4a0166f94663789fda23681c790469624d8039482fcd91c0dad48e1445dd2620", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 4.463307913974859, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:33.292871+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "9eb91206-836f-4787-8d1d-226b9c461fee", "result_sha256": "a5d07e91ef991ee59c3bde1d38b70e5b740b95cf6de9751f28efa6b2952e5de9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.751092229038477, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:26.787020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b97aa521-2439-4724-881c-ed263c1627b9", "result_sha256": "dfc5c3934924c22f75a3299d170952faef34b43cb2b24f5333a0b2acb7d689ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.6827032970031723, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:47.633268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "60ad0a9c-ac25-4e29-bea8-39ca8a6060a5", "result_sha256": "8e97b4340bf7551bbd87ec65c28482d7ee17c8696b54e18d0499d7528d28cdf5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 7.176943654892966, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:04.213893+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b29e9f4e-8471-4b9d-9f05-00a61fc69238", "result_sha256": "c2a1780e14312918149fbcd7658128c42305eeb24ecf4e0c244334d1c7b498db", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.6852307179942727, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:09.983307+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "acabd9d9-013a-4d01-ad39-42c8e753ff85", "result_sha256": "603eb8be03d9b1250bba0b90cb78890342175eaf7d45457d9eacb046136d41dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 4.340433359029703, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:47.455412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "79f6c77e-924e-4044-aa9c-569fec8f99b4", "result_sha256": "351ec5224c420e0ab3a8aedd4b7db7b515a4cdc1755f75f9ba061b3221181167", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2780, "key_errors": null, "latency_sec": 3.957286187913269, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:58.729252+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "fdfe7e38-8490-4760-8d41-be57515a895b", "result_sha256": "701f50d4286754335e9c1fcd2e97c195226d2eb47e7821e046683c0e08fe87af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.8117207209579647, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:48.803936+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a75c4b5e-31b9-43d6-8eed-115aafe94f1b", "result_sha256": "76bf7e150132d26172568ed104e0fe104655c4e801b27614ae1259f4d8a54ecf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 4.265403012977913, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:14.236503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a1233e4a-d061-4c44-9dcf-ab700dee8c89", "result_sha256": "637e74cce83247f2f906dc99df40be7e334b462e1d81f5b1010e5dd0638525e4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 5.46878115599975, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:04.421577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c2d9b039-fb99-4f4b-b3a0-afdc8596dc67", "result_sha256": "7633fdf20839576d5c0d9e81205864c38e512c784209964a1ffe05fd719399f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 5.662840030970983, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:08.737005+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "542693e9-8671-442c-8d4b-7c3df359db07", "result_sha256": "b37004d7cda207c34120dfa1f15e3155e4993823382aaa6ad7a7378a7d9e039c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.175597527064383, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:19.645218+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "dab887f2-295c-4a3b-820a-a343fb3dbfa1", "result_sha256": "6745b9a7d5a124e96064ff2dd76d650dae79ca9adc129782f81a009fad306304", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.950038366019726, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:13.388043+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4bf58c52-8953-4460-97e0-7726d861b855", "result_sha256": "eceb74b927679b4dc2d73e5ff888fc26c24417066c653cf3f20e0d4a64d1b94a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 4.405252791009843, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:55.760204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "74a3b787-268d-4d12-9483-03d805e6e651", "result_sha256": "073d29206a3c8883d4d56c26cc91b17e1830ac0655f998a9fe72fac74437ca13", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 4.0052507689688355, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:51.655455+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "cb83498e-0576-4f0c-8522-a3eb8757128b", "result_sha256": "f2eaac021a1fdcd210ffcaadeb3f0053766d6d106fbd2fd037dfd27d06525f3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.39313070604112, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:24.725147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c321d115-87ab-42ef-9bd2-257d1090e17b", "result_sha256": "3b66ef9f1aebcca8174bf7d27cf0b84e8c01e12e3e689ad30eee3e1978d5a45e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 7.295461642905138, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:38.020713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "109fec6a-fd0e-49a5-a7d9-644eb6f8a6e7", "result_sha256": "ea2e3b8d476b5dc59e08755c2a9655e0ddf5c2ddf4e449f1f8dca16f6451c174", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.531889377045445, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:51.254988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "30e0586e-a190-4b23-859d-427b7ef09ee3", "result_sha256": "9bc54794cbaa670f7bfe9d94b75b958dd4a73e56b3ac373b5974c42e6d00e13f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 14.002781323972158, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:47.942054+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a36bc74-26e3-4bf3-aa1f-7b150ffeb536", "result_sha256": "9cec2566aec6f5efff1e3ec3db80738a276577d068d138d8ed7417a6c43dc648", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 6.368188295979053, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:57.820797+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "51e5a6c9-9bfd-469b-90b4-ee9e4bbd6ad9", "result_sha256": "28e6b732c463098fe6f426af02478a9da3386adef8645bed16953114d8d9a3df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.2013862510211766, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:36.697723+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "16586658-3f52-4069-ae2b-ef7c20146db5", "result_sha256": "b84a0519e03dff7977b857e84cbb8760afe053df647a4c04d836302b02a2c14d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.1109944520285353, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:37.846785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "dc376bd2-186d-4a26-979d-6b0b86b72fca", "result_sha256": "3bb02713968b2bd80e3c6472f286b3aa4a0770e4cdc1eadf746859991b333fe8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 4.046527134021744, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:18.651464+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b788d835-4f88-499e-8b3f-3b9f381ec04c", "result_sha256": "f1943d9c140d4a19d6ce57d99d25dc8cedaeb7e99d060506929ba1e59b0f6424", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.535837634000927, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:58.348683+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "7e572b61-4b96-44fd-8031-16652ff02e0d", "result_sha256": "356ceb3eda9e2c845a5429c04dbf18e52a510f2f3f3f7a9420b29980c4fb8a27", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.5819608939345926, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:42.790720+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b078b28f-4e38-429f-bb46-3d2d071d32e3", "result_sha256": "3803b9a563e22817252fdd723b7a06dbb2319181990f9f741bea7c156187b2af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 5.154253747081384, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:56.599283+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "6434b1c8-8cc0-48a9-a87e-94b35e25175d", "result_sha256": "570a7ee9facbe66fa19ff57c36064836b622ec892b8e09c4276d769e0ff5c0e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 4.038416246068664, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:06.184254+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "34660e76-67b6-46de-ab39-b08b8dd8d39f", "result_sha256": "15db70baa3017cc2454b1550a994b26e9188321ebed93d5d2bbb40cdd1e1cca9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 5.054691047058441, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:23.217175+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "2a565be8-b936-4d73-97e9-5aad32aaf06e", "result_sha256": "9f451d2935cc926817904ca92137a5e6727319575d51f9df200f72d32f9d8f22", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 9.548461873084307, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:42.937791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "73a465df-3365-4b86-af11-3027c0a0eb0d", "result_sha256": "748fbe9b019520aa398654298ee848a3eddb0dfe866e0c52f253896f25dcff2a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 6.338730292976834, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:29.706150+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "1d30d8ee-6d7a-4608-8450-3c7e69771ccf", "result_sha256": "7857280837c50c0a92322ed8edb68c588d0829051ada99c5fd301a5472803800", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2822, "key_errors": null, "latency_sec": 3.5479685169411823, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:24.621847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "4afc03e0-d916-45c6-a9ec-3acba2e21e47", "result_sha256": "970f4cccee1e314f57a1ac4557260f1b4ce0ffbd45eec3af33a788cd5c4f4b2c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.34079084394034, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:37.331097+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a9db81ac-b29e-4c96-82be-665c310fe289", "result_sha256": "da0473cdedb86cf3b7e4d17610f247516fe1fb5e0e3f32560262018e7a92f219", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 5.866790223051794, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:43.790216+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed66e3bc-fb57-4ded-a6ab-f4822cde45af", "result_sha256": "60a1227b8342beb30de3963af12bd0ebc20018550068ca1124c312f5cf2c4269", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 4.3933457439998165, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:52.279711+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "52cca9b9-8b32-4d5d-8849-5959e7929658", "result_sha256": "af7181339da945cf780252082af544155a61c9fc2c2eff76f7a781808265469a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 5.405033276998438, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:40.005945+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "6cda24e1-f237-4c8b-a2bf-d5b0f0c75340", "result_sha256": "7506ad0b586f68ddc46ee2d32031769c42b10c9fc86903b6dabe4bc9a39f1804", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.76550324796699, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:13.975498+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d816157c-d7a7-48f0-902e-c432c5eef9df", "result_sha256": "7337576b5950a5b0d49833676017e1633f1a1fd3cf4b70bfd4f011ba3d5364fb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.6987355779856443, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:59.845997+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "fce462dc-8b4d-4537-b932-e9aba861b91b", "result_sha256": "6a097bd6aafaeab917ac5af563ff4ac41b3f26a05d1938c67b556bdb1fc2d506", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.58432379597798, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:02.423822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "fbb0dee0-3413-4e4f-abc6-1f1a767d3189", "result_sha256": "4eb49f3465ac359e59f0bce00746e6690f9307189c2b229d799e038378fe59e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.452184319961816, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:17.217568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "53cefd7c-1c34-4e9a-8120-b7239855ef45", "result_sha256": "795c49ee9cb689feb5229ccac25c50ef9d05e63cc85dbe8429ac6b6911fd9eac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02877, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 5.465248306980357, "model_name": "claude-fable-5-1", "output_tokens": 15, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:58.929344+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d93f45be-f1ff-40f4-8002-17e4f848d62a", "result_sha256": "00918b89837311c40935942c74bf4655da4680ff7914b140e467ee3cdf42af9c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02872, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.979646266088821, "model_name": "claude-fable-5-1", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:14.996983+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "d64ec08d-fe77-48ff-bd6d-5aa0991a1696", "result_sha256": "d64f1efd2f19b44f5a7873626285e7f8450dca5df089484eb440da4202473802", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.4346824520034716, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:43.378158+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "727195c9-fe4f-4c35-acb7-94d41201c2a2", "result_sha256": "f55f412cc06065acdcb3d2c0df40098941ac83d3294669ba67cf1bebcb5f5295", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02857, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2802, "key_errors": null, "latency_sec": 3.469360061106272, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:15.822767+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "00dd3bc2-b5bf-43b5-80b9-ac492400aa10", "result_sha256": "c12f26218a3faf29d7d10387523d38751a06abd9c2e37dded036a19fc3a1e93d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.580396836041473, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:58.684812+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c46245fe-dc50-444f-8e4f-716c8962c734", "result_sha256": "ce9f852b7e2d40abb8e6dbe4707ee798fa132fbae64d98905359e4a9654cd7da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.615574129973538, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:33.192974+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "46778cde-8733-49f8-b63d-3adcf0eddfed", "result_sha256": "cff0920d5803cfa2d45125c472c5e1dba33b797177bb0199462532f26ce1bbf8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.60788536304608, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:59.182204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "0830a0a8-47d1-4889-8d54-e71d053f8ea8", "result_sha256": "3230eb9a8bfb3ec4f0d10b3a65f861a9ffa8e04504c29f6b93e7dbd6927b3e94", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.328210109961219, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:34.754364+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "ffa2f578-1504-4e1a-9f4b-a89ddf7bd066", "result_sha256": "71105b6910410d13348aff3fdd7710f4f69a46f2b5e3de6e10a148c3b9bffe2e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 4.457491965033114, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:44.191601+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "89926d21-c595-4a88-b1d9-098ec424d1ba", "result_sha256": "8492626e621bbe715e683beb7ce5fd7ae73eb9fce0d37891119b542f4280ad38", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 2.999300320050679, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:39.410259+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "13a6d559-b118-43f1-af33-966feedee1a9", "result_sha256": "97956a75aa8b1aec8437a3ebbfe5a607863c7825b01e9469faeedbdf50f48595", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.1501159869367257, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:52.921880+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a4196c89-749f-4559-9148-31d7701836a5", "result_sha256": "74c3f3f28257d8e954715eb07f9a826814d5c751b8b1967b7b35fa01372d4f32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 4.009503436041996, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:36.459955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "755f507c-0a22-48e2-818d-2ebaf2b676a5", "result_sha256": "7b4e5f2c48e0c202dbd3bdb1ecbb5fdbb0563b931e2d57adecfb314fa715d6a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 7.6905029020272195, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:07.900353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "11beff31-6099-4124-a95a-80c932db4c91", "result_sha256": "7379902510c11835f9548e115f70ebe550b6e6eb0d33bb61543a73779cfdb070", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 5.085992314037867, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:56.545569+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c4368e3-cd72-4ed9-8487-7b2b4eec3764", "result_sha256": "866bf393a770e30ab0c833b8e7c0b6049a26f60a3a49524920e260d1dbfda733", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.5782261370914057, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:42.511620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "35b64d89-aa53-4445-b102-b321466a29f4", "result_sha256": "385f69f38f45897b1e22bcbc682c321d5a8033a961f794fbba37119e7c89aef8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02871, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2816, "key_errors": null, "latency_sec": 3.750715372036211, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:47.527403+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "34637318-1f12-45af-a900-c16596a2323e", "result_sha256": "4388fcfbd2752b32bb428a849504beb2ed54e78b5fb7fa1439b8298244116726", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.513915000949055, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:05.714413+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "553570db-6344-4072-946d-5f60682e6309", "result_sha256": "940186bffb721fe19ef82678a4be92ed451124c9c08769115909fb2746b977ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.3665910359704867, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:32.870436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "2fb70a55-ba27-47ee-bab0-db643ab454d0", "result_sha256": "6b6bd67b85d8c727203ce0c5b69b9033e2a5279f1a6c71d5da810bf2157ea4c6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 4.137856925022788, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:51.031051+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "650637d6-d149-4a85-b24f-4f0df637dcc8", "result_sha256": "e179bd0204486c040266a32371f20b3bce90619aeaffb511428225317ef3ee60", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.261525260983035, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:27.517262+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "c45018bb-ab77-4636-b3eb-81fe456d677a", "result_sha256": "9983bb9959cba4526e3f3d1384ce26a74caa00030fe0829e84c86437be7db141", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.3935657660476863, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:02.582880+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e71d711-7ea9-46ed-8b47-bde3e01bfc2f", "result_sha256": "ad1493b0c33bf262dedc1769c3dc00e2ee1727cb35c98033d8a0088f5d114fe6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.650585123919882, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:51.225517+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "ef8f3f23-8254-4c97-9770-2b599658f009", "result_sha256": "6fecd24500e81c93cd3f60202abc34589cef9c07be8e7e3d52aeda2d4bcda0d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.6167358650127426, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:18.794400+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "418951bb-a792-41a1-930b-351738e4a717", "result_sha256": "46ab1e81288afb9cf4debe3c533209387d289cdab7a7a28ad095690d51bc5386", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 2.400648330920376, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:05.455799+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "b748f005-1c70-4b94-b6e5-000bf9b7269a", "result_sha256": "e7e4149691d37ee0d121de116531154381dcd64397654df984651376d2dad576", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.556522549013607, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:49.510357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "edb22c93-6aeb-461f-a371-21d6db197c16", "result_sha256": "e12d465cea9d4efb927bdc62776575244a40ad0ce8aae07c7bc88aa57a9552de", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.6378387480508536, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:49.776422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "9e7d13b7-02d9-4930-a593-94ae4522474e", "result_sha256": "29c5c517a70b233fcb265ce76b319e93f8e5ae71c5baa0d1091aeccd164f555b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.118125506094657, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:54.752989+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-fable-5-1", "status": "final", "included_in_comparison": true, "source_attempt_id": "a10e8330-a0f0-4fdf-8309-1de175b03a1f", "result_sha256": "d45b5cb5c6890cc9f49062657dd105b3f84bd70561a780e339f7f0af1243e20f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0284, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2785, "key_errors": null, "latency_sec": 3.9625481580151245, "model_name": "claude-fable-5-1", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:43.120362+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3f8f2f2-83cd-485f-8b49-32b4cd1a8d10", "result_sha256": "809d779be2366acec91883aaf91a77f18338beac4aa7fd7ddb8bcd69a14f344a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.5904615969629958, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:16.936659+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b7d7fd1e-75e2-40be-98d3-575872c33155", "result_sha256": "af356dfee5febb4ebcbe405efa07fc2a6b2132ad81a9bf9fa2aec7f197d1f48b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.5659048949601129, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:13.080813+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b99c0b5-6b15-410d-937d-4a8ea15c4060", "result_sha256": "f2c60cfbcfebeddfbacb525d99a874230c0f4a8219cdef3889befa65b8284b7e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.6878554030554369, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:07.393522+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d7ac7491-1e7a-4efb-beda-5ed691b54105", "result_sha256": "cdcdc0ed446f5d512a2a096ddbac6e97733b030b2118487b8c8ac84fe7d9ecf6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.6815145130967721, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:11.215395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9a335b98-abbb-401a-9db2-66626b7fc5eb", "result_sha256": "532376d818ef8b72fb670afa9e66627b5290b1517aae7afa106b97c6500a941a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.7168398959329352, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:35.960907+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "efc0c185-a622-4fdc-aa3d-16890b7d682e", "result_sha256": "e2f18efea16fe1122902e98125ffff2c5a15bd0c1f5eb959732bee53b792136b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.6704592190217227, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:51.609459+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e601994f-2ac4-4b23-a6b4-aa2b925c93f7", "result_sha256": "56b639647c08dfb0f62ff2a15809e21c9389cb56a5c5724498611657c69de5d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001857, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.7300039129331708, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:37.786779+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c1d71d7-a8dc-4109-9b7b-0438c3adcb49", "result_sha256": "6e8ff1e7a1915b6bb423353456bf837b8f974ea6fdbb749894cccd58707699da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.7340721199288964, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:20.446630+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6ea3ae39-add4-4209-b5dd-d7c0498e9a81", "result_sha256": "663e48245c7275c44ecd33354ac06bf8cf4ab96c25651ca913900515057c7b93", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.6986314459936693, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:27.980176+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1c658ca-510a-4057-a31a-9bc2c9866d4e", "result_sha256": "ba8d4fd046c445fa1112ccf190c0d1780a9f0606de6ddc07eb8e786bf9034aac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.6583242990309373, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:50.406745+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "50eb887a-d34a-4dc4-9bfe-2d65133e782d", "result_sha256": "75a78a7e08435b736b1e17453542979c293277d9c4ed13bf736ba2022bcf1ab0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.7038172039901838, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:27.524498+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6f0c3fe6-42bb-43bf-b018-8d21c97fc637", "result_sha256": "deef27a6c6f593c4852e1176f81f0a31469a90ffc534b8a312ac392533c76f61", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001862, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 1817, "key_errors": null, "latency_sec": 0.7913193010026589, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:49.434044+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "943162b9-9aee-4757-83e6-e7598dcd6e68", "result_sha256": "1f0676a6d06e0a63badafdedcf860942c4b419394688508b58b984620e4f7cde", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6952014289563522, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:54.074783+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d829b07b-6c66-4e5a-b1ef-b6ab66f21a7f", "result_sha256": "12997b90109cafe5741b6627c203be16f20445dcd44eee9461a6dbe31e72b5af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001925, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 1.4491039759013802, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:59.733120+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "da30fc66-76da-4ef9-ae9e-73bad3f2e4d0", "result_sha256": "8b2a00014ee0b99d79aece124b1857e4c330ea5689f233619d89be5ad0b5acce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6548922980437055, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:58.756384+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "82408e73-b2a2-4438-ad9b-0c084563c565", "result_sha256": "541003f90176205ca6723f43b85697f9ce2284b6a128083ad7672014ad8f21ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 1.1583858800586313, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:04.512553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3d87ff0-2c81-4537-a4dc-ad8630578327", "result_sha256": "b061675287bcddb2616a6654430657f9b495f58bdc237d517a2804526c71dd69", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6392772360704839, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:02.667200+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "03471440-7241-4b9b-a30b-2944d544edd5", "result_sha256": "1a3078f2a9f8ce2293ca37f902cd87e4c3d2ae5195033aa9687cf0ec530538b7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.5818098959280178, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:22:08.698996+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "855c2dfd-b287-4908-bb3f-b939a613b6bf", "result_sha256": "130b26fe6a52df952a4df12a6c4de17ec7076547ee97409b265142b7273344a4", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.7020874910522252, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:40.794304+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac9895bc-1899-4bd1-909f-0c220c47cc58", "result_sha256": "f2e7b41b2629e2e9ed2050fc3903f3ce2425fc55c9f7b447efb9f8ccec9f817b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6763078770600259, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:40.762333+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "3f0efe41-b9f3-4f00-9a46-683736f3af13", "result_sha256": "b68019ca8ae99a8102dcbb3708ca467d6a1681f4b8455953af4be4504cddb7c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6084565379424021, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:10:00.200210+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "3370a7e5-5925-423b-ac2c-471037ed4282", "result_sha256": "1ff2a8bacaee2446576a349784a14626cd899e4c0cbedc6c0abe51a914f231a1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001925, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 1.1935201600426808, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:20.594144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1d977ec5-c1f4-4f6e-95d8-a6c7d9be2e4b", "result_sha256": "8c2dd97f1f03a71a622230097af0a6d94077ab47227624ece8b8ae2dd68c7013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.7179965369869024, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:05.162398+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a08f5a4c-f48b-4105-98ff-8499f6808d10", "result_sha256": "e6c72c1f7a08462fb7f13695649c220312332151267709a3f6254392ab307d5f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.6358918979531154, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:48.202141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d51a6d60-3275-4813-9e77-6e7f96a39a28", "result_sha256": "c7ab294c52e5cbb3ac26615dba64930c588e2898d06427c278ec31eba056e890", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.729100083000958, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:47.425355+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a277a03c-e14b-4153-8bcc-463d98a03215", "result_sha256": "1dd764eacaaf893419c31d0e5a238a7361a94f25556f0b546b509b07a3b87950", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 1.1171778429998085, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:00.466723+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6eb3ee54-d050-4d14-aef3-1b6a4c0d78d4", "result_sha256": "fd112eb17f34142f2a877fc3e224148fda99963b6b45d1ff5fb4de15d57051d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 0.7701321670319885, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:55.521989+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e85cb36-e61e-4f40-998e-21f52262c783", "result_sha256": "f239834af309d58b120b62cfb83d0898725984762a3e2c78fa853ed0311c0696", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00193, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 1885, "key_errors": null, "latency_sec": 1.0323586100712419, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:42.417168+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1af0aee2-ce35-44c4-8a83-6e467dc72984", "result_sha256": "abacf2201cebd676563a4ea7bd9d020f992565a4be00508b53b05707fd59c8ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 1.0003709819866344, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:38.070397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e4400fac-5060-4cb4-a008-a3c17a594f2a", "result_sha256": "0aac0254143dd372e00cbd023654639e6c085207473c14f2d8b9b3098bcc64bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.5399056329624727, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:08.079101+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "8bd1fe07-88a3-47a3-8f69-d2ba20179403", "result_sha256": "5b77ea6ddea2bd2c342d23c929ad2b92e174d6ab1296fd59833162fa64dbe67c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.6068368430715054, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:27.546243+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "0ba176d7-1e34-4b8c-aea6-a8dd754064a4", "result_sha256": "cf37a54aba7d16498a6a302db53df2e08cefea7949fb129de35ba5a72e4b7fda", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 1.205817612935789, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:22.809891+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ef4410b4-297e-43f2-8e88-9d7bcc770124", "result_sha256": "9352a9eb5970670812e96838de0a225704009c562cf38807c550b7c4d92d6c68", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.6339506580261514, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:30.912076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "0cb36d47-b473-4692-b4e6-84cf340b9081", "result_sha256": "a08544872fc356322a10d5a0acf7e604f2d6b6eff84fd4638555c7565ace86c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.6934561929665506, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:57.710976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "55ba7698-5603-4f4f-8bd4-2da451462c96", "result_sha256": "b9f9c96560797f8b5e9e45d812ada63ce972a7b6aa7cb6b5a167bc04530e9cb1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.9619470369070768, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:39.766771+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f2459041-8bf9-4e0e-bba4-b462e0f533d6", "result_sha256": "72347bfcf30ae3a2ebe68f3991e8bf9e07028d3693190f52c45d116f42171972", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001833, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 1788, "key_errors": null, "latency_sec": 0.6471397160785273, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:15.060826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "bd145289-0ac6-47a3-b59b-8f3ff9081431", "result_sha256": "2299fbc4bff2543d6282e1a48ffcd3f6c8f8c0e99652f5e984ff8178dd5a1773", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.722560988040641, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "E" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T12:08:54.406660+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f97af48b-37fb-4901-9af4-7815e8fad9ef", "result_sha256": "dea4dfa742f7cede5be66261268c8280ffc0e6196b40144cf2913c8990c68428", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001928, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.16552681196481, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "A" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:35.842616+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "cbd6c397-c8bf-4e9d-a9d7-6181c9142208", "result_sha256": "a4a51d67ea4a6fe4f70adbb300dcf9da285ef21650db0b93c7c35e6b33b1e9ad", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.397491391049698, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:21:54.253027+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "fa4ffe96-a9e6-4efb-a19f-ff62932f5888", "result_sha256": "affe9bb3f170ee8d3bad7197ad921f276fa1aeaea43ef553fd10156c30a2d1bf", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.1420170350465924, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:53.979854+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e061d98-c826-4d1e-8328-671803093fc3", "result_sha256": "3cd5967404e110f3e2120710566283cec4c4a57c67b32f3b6515187d97a2619c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7548836969071999, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:12.828570+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1d5cb89f-19c6-443f-a075-cdf7fab9cf6b", "result_sha256": "9dc6cf9847dcf8ab1342357762e8996b6894fd2c2caf0a528761783a2f9ad3ff", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.3341470110462978, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:30.086529+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1847d5e7-5514-4b6a-b490-80863f72ff93", "result_sha256": "5312d42018dbddd9167a67f1c2c285570092ce205b72cad53943f3f5071c3735", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.339963307022117, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:23.804496+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "18c54f7f-552a-4f4a-a40c-3ce1c40f8372", "result_sha256": "4111444b971d7e96061da05356d9d0e108121180c86fcd4b5e7f00fad3c9fa17", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.6942421710118651, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:14.000577+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "84db5215-7fe7-4fdb-98a4-874c5ad47fd9", "result_sha256": "f0ae597befaba5a205292f19464e097c5f01f31887a64adc5c48b5ad94df735f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.231770273996517, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:21.400400+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "65e57cc8-d4a4-42c7-93cb-5d25cb5820b6", "result_sha256": "1855e38b8cca05df72eee894b35758e65fcb347059b0539ad4703ca401455e61", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.4027256059926003, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:15:19.689571+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "98cd9d97-18a4-4b00-a5c1-f6758a0bda93", "result_sha256": "1cfcc8de0c6841a7d6ed275cb972ed16fca8cc7c505fad408e1ea61409554696", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7028968119993806, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:47.678828+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9f47834f-8f62-45ca-8431-c72ab1715763", "result_sha256": "ec2baea9e3856c5ca67e6daf1fef524ea96d0fd17c93e0bdd0ea7c4442f9fb5a", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.3179214539704844, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "E" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T12:09:19.526335+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "4586b074-ab67-4636-9bfd-bedea500e0a4", "result_sha256": "f5f1fd2b4e9cccebed7d21a89ef44ffa81bb1da5e8a70dc3b6317d16323406c9", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.2004007059149444, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:45.380284+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e4953d71-f46c-419d-97ab-140a53cae304", "result_sha256": "3acb8f08b90dd91215d940aacfde680428e407c6c29737bbd204cfffea30aecf", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.6779803119134158, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:17:41.422161+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "71d99db0-307b-460a-b0c6-4c5f22b4d7ca", "result_sha256": "712404398ab63da3f28126c1a2ab6687c79a55522a6780d51114b0af3ea0c92f", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.283471809932962, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:37.126474+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "94112b63-410a-44c1-962f-5d4a436890e5", "result_sha256": "eb1ec39362d711e628493308f2366e0fb4ffe35d62fa2f8a3af3f060cf0654ec", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.210386881022714, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:32.118274+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b5611c9-cdb2-428b-ae4e-a693780a50be", "result_sha256": "2f9305ecdd2c6122038d42eb67dc9c415fa8285472b4c0906adc120c2db9882d", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001928, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7947205269010738, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "C" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:37.982096+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "3636b37a-c7c8-45e9-a083-3dcdac3767c1", "result_sha256": "571cd7856331a987fbe5c93a5ed251a3711bb59b001b62870912245db2e49434", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7449686949839815, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:22.641195+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "169ee1cc-7c86-46ce-a59c-f81b4a728720", "result_sha256": "9eb46ac3c7bf4a196f8e074fa17ea4db63871efd5a03037e7f19a631a66023fd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 1.2700655579101294, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:17:00.842048+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9be4fa5b-943e-4f88-9987-21341ce87fe1", "result_sha256": "9a2c273bf1dde87d02aa2564b90a37699ec0311ba56b7accb825c47ec61bded4", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.8097340340027586, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:59.746713+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "83c6afa5-31a3-41ca-9eb9-42cc1030363c", "result_sha256": "ae1cb54b4731173fd072271381150272092fc7571399ef35fa6c361f20670b48", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7256684509338811, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:57.902737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "423e36c1-3519-4221-9d24-6330a9cf8462", "result_sha256": "0162350056351c448a3d651056fe8968127c46a4daf302ddf38e25b1caf89ee8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.7052383120171726, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:05.140568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f5fd9d2-649f-440f-9e0e-1278e2083007", "result_sha256": "89ad0448059cdde51ca26b01f25ace8959ae26083c001daf1ca0e3612adf6c34", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.8302049450576305, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:31.467942+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b1c7e28c-e1b6-43c7-8014-5386fa7dc0a2", "result_sha256": "47334eb36c5a3952760a0f4c32a2c24589032b652ccd2ee64a8f10a54bce22c6", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001933, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 1888, "key_errors": null, "latency_sec": 0.6640378150623292, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:51.833687+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "937ef1dc-a373-4254-9d77-bcb45356c460", "result_sha256": "fa9547bbf13424f5d0180cec51c5930e44faecc0ae24dfc5e2dae5ce1b2fe363", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.5105005570221692, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:06.099855+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d06ce44-1c8e-476c-bf40-a6c25e7cc912", "result_sha256": "66edb228683153251637e385d416f7b6237e14b22b93376c4925a11ed8a77f7b", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.6701229079626501, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:56.500454+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9547d889-84be-4466-b0da-a98c9660bde5", "result_sha256": "854be4f96040db4053c36aa5adbaf111becde72e4680c47e5a2a8a8e2ee07b3b", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.6930869950447232, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:09.058939+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "60ff5737-8cb3-4f6a-9bc1-ba4a47d433f8", "result_sha256": "fe85f3ece9a76b2c6cb1e254a6565c0f68cb46445b9500e4de0a10232699b3f3", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.5182345689972863, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:49.970243+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "36226aa8-9b58-475d-a935-29f0a7d2a854", "result_sha256": "5339e76856ea8d8047a05ab1bffd2639d5f8a402ad1167423875f6e822beb217", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.7802846509730443, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:56.821775+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "00965604-cfb1-43a3-8737-c0534893adf5", "result_sha256": "3f6ca9a3555ad3b8734fe0209ad4acf8355e08cdab7eb8efcb85d9b478b32e73", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.8700868400046602, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:57.920516+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f362ee1d-a067-4432-be4a-0af3c87a156f", "result_sha256": "4ca0ba62b932299be46b8b128c4d662e5666b9b750caedb1551853df9d83129e", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.7185355110559613, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:16.612244+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "0d0f4169-5b91-44bc-90c1-ac6ecb095ecf", "result_sha256": "3ad5b01cb118d5eeedf28563681a78cd4b8408176510a54adeeecf278094ca27", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.7597760929493234, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:34.340151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0eb856f-b470-4494-abc1-4a52cc1709ff", "result_sha256": "83f00c1985b83fa5219d1a92142e6e58e6fbe619268e56c5839cbd6b79ea5463", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.6171250379411504, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:04.767642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "236e53c9-bcd0-4335-aabc-2c75234c369a", "result_sha256": "8a11a9b90ac5c81c472ca4bf19fe92be8dd2c19c815906dc85fb53eb0473d573", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.6757539740065113, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:31.121341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f0ec20a3-87c8-45db-900e-3e5cd3668943", "result_sha256": "c887afa08c7610fc2a3a17ce7b3a09e6312077d9225bf4a6487c840535e923e2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.875006131012924, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:16.020556+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "49945dda-b813-4b30-9014-9c34f2eaa547", "result_sha256": "bf3a2702eefd969d87e46071ac4368d09e6055342cb2af8e87ce809b61b7974e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001852, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 1807, "key_errors": null, "latency_sec": 0.7813130930298939, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:48.053609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "76354ce4-0985-449a-a9a0-edcd969c04e1", "result_sha256": "88413a433faad2ef2ee5419929ef1e2c69b5a0851703f5b8654a5c018be6c448", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.8098154990002513, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:07.664390+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "231099ff-7614-43cc-b5ab-9ae79a7b6106", "result_sha256": "09feb885862f8709d59ab6665d3abc919d7dcb444fbfb4ca0fabc5fc78c76e3e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.5463258699746802, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:21.635490+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce780fd7-3836-4eca-8922-ae1d2711ca92", "result_sha256": "24faa750a39dcd3f87f13f4f2bfe4c8af661ef43e3e933eeb26b84f125ba3621", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.1542429350083694, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:38.799934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "dc784b5d-bb86-4c39-b0af-0176e0bf46bb", "result_sha256": "fdec63584d2d64d66f2092e2364da0a62e0a98f80f41d075c26fc14dea3e66a4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.5370337018975988, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:40.201008+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "dcf50596-a699-4bd2-8623-eb188176f6a4", "result_sha256": "926821aa0d165b0b68882839b0eb5162ea7f87699023a5de346ca729e776e3f3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.6312154530314729, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:45.692306+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c5b3ae2-ec4a-48b9-a90b-b2d6c10cf45e", "result_sha256": "763e9456052f5aa22f12408f854cb60e6dd56bac54a3e428e69d5c40a66915a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.2964780090842396, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:41.202065+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "48cbb426-4cae-4db6-9ab0-3a342c8d2c7c", "result_sha256": "25f1917ae1bbcf4d8157b8550e2eee6907ce29a771c67607c447f445cdd5c1c5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.7949462640099227, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:53.213604+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9bb50283-ae5c-447e-bc51-f2f54c88ac3f", "result_sha256": "a3cce113bcb3ed4044e3a08c2674e97ba9d29499cc321c88eb9c2856bca9b6e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.1820595170138404, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:57.835291+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "17768288-de65-4ea0-8059-f1cef7598819", "result_sha256": "3a69cff6f60073ce4eae7ec5ceb9fc9804094bd1b2adc45aaf01042809e6c933", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.7383290559519082, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:35.529006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6472dccb-7ba7-4794-a8af-8dd656507c3d", "result_sha256": "535a4c08d040b8e0a24e5be218877ce8393e996b1898cb6fb112dfe0d64a6cb4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.1149664520053193, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:46.703944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "86ee5c69-6ce3-49ca-9def-6f3f899b88f5", "result_sha256": "74e1fcba2c12855e6eae88bbf400bdeddf33c228024531993798e03a92cf01d7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.622902411967516, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:31.327915+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a4c5b8d9-1249-4d77-8b95-99929d03ae62", "result_sha256": "ff7599b717d4584a25579526c95528ee31e66a8dda328acf5b44786bcb916869", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.1723639840492979, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:07.647666+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f7bfcc0-e462-4c63-b337-23a3311f10a9", "result_sha256": "de8e04a65832d1cf4f0f1b8ae39fb3a9f674b0e953e00006a91d62fade088ce9", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.7180613810196519, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:01.867850+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "4151f796-517d-459e-a5d7-2b5682326e3c", "result_sha256": "b3510d6954a5d234fc73c18bc8b45b884adc712c3a8658a9210d1548f9f14c56", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001851, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.590163360000588, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:10.642108+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "19fb8c3c-402a-4b9b-a640-914ce5d5e293", "result_sha256": "e757aef5656816eeac264a62bd3a067dccc7ef4901dfe3a11b041880e9da27cd", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.6951365780550987, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:10.296630+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "cde55760-168e-4b21-94ba-0f1a976b887f", "result_sha256": "07e439ea0353f9b0592746592cc363084e5582658e9425235368851b1883c09b", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.6840691700344905, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:25.836048+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "34a46cfc-e4a9-49a1-9dba-a5517ef4652f", "result_sha256": "6d9b252da464a2ce51d819ad6738d7ff9fba135798e372c7c4d8a73007b9dade", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 1.0519587270682678, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:27.000305+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d5eeedd-d337-4b19-bc2a-e331b06edc21", "result_sha256": "08cbb43e0070d806186e9951fd2f45120fa35d558e16695e422f50c712b230c5", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.6989776190603152, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:36.915776+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a80d24cd-46ab-4b38-bff2-8887feaa1cbe", "result_sha256": "cb34bfafe052b025e4f352dea68e294a5c86c0d1a0ef830685155e951c74cdc2", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.9970288219628856, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:56.170807+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "bae2ee9c-9bb4-4052-b987-59b009771376", "result_sha256": "e7b5bc30941d93682f4f74666a5a71892b628cebcc6ef0ef4401200a06c060cc", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 1.1825741119682789, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:23.490549+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5a679139-aff9-418c-9cc5-38621fa516f6", "result_sha256": "58b6dcdca94bfdeb93c8a5dcf0c00b493051ae93ca415808aa141b68cbc8978a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 1.1949724860023707, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:36.884315+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "369f9c79-d84c-433f-88a8-1155f0077228", "result_sha256": "2c2aa331a9165e3ebf12f1186fd10589b0a48bd6332d8bd6112aff5959862c9b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.8553624099586159, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:13.200565+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "807e4370-83fa-40d3-84d6-eba23fd257a6", "result_sha256": "1e46ae2b7fcb35632b3e4e6c6ff6a92a4e1984cd4548d527231420d968a10841", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.689378670998849, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:05.170905+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "53fc146f-32be-420f-8043-e171440a0738", "result_sha256": "8a635c0c093caa6862d980c57997dc69a5095cd226443d138cdfdc4d5d39a266", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001851, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1811, "key_errors": null, "latency_sec": 0.691694593988359, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "A" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:17.320601+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ad218518-e157-4ec6-aed0-841fa628d9cb", "result_sha256": "f83c590551218df3f3f3df729a577327128493b2d2e99a08ed86012052bda532", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.0914885730016977, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:03.509727+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "dc3d6ec6-2eb5-41a5-8410-0def86773545", "result_sha256": "63e90d3db0b4ca3bb24dd4f3281743119edf0fa81d335c6b37024cd49d7a321b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.9663203089730814, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:09.340991+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0f78299-335b-4c97-905a-0bfd41e1fb09", "result_sha256": "632b29936d31d992d2de98043771e8b62abf7cc2d6d5b065d25781e4a133d89c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.0822979880031198, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:00.494239+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "83b0c289-2f88-4afe-9e5e-82acf8a0881b", "result_sha256": "539787508d44a0de4b1fb7527eee1889bfddea228ac2ed572297d81f5fab88d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.6856301290681586, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:50.564221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e9c18b9-f10c-47e5-a723-29c5fe05b39a", "result_sha256": "45dc6f7842dd48f434ca4514a15af0629e663b9e648c02bf99c8bffbb130f260", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.6795538100413978, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:29.674278+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f18bfe69-ad28-4282-a651-2098f66ec87c", "result_sha256": "a61c1be2afc70ad09ec0c2c2f1da0af7a17ef6492d2b6c901595a06ee4c39266", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.679461546940729, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:42.378544+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b9c7a35-7530-4baa-a1d2-7cce86fdebd0", "result_sha256": "f0e65581fd27006e032e1690ead674a9923120351051dbd973430532f42291cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 3.843994616996497, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:52.402626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "919cb3eb-5125-44d8-ac1e-59ca421c60db", "result_sha256": "c5ee962d35eef5b0b71527c1014d44fdc0992a422881f36084de79281e966603", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.2248770860023797, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:02.451957+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c00951e-7266-4f85-bbb1-d473835c7635", "result_sha256": "5e78f182ce97eff8d6d9d536b1d5129c1d6fb52086b625c934e7bd4f1459ee63", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.8001180819701403, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:27.273428+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f4c1801e-0966-40af-9099-6f8a328dc7da", "result_sha256": "52d0a3b9d10a61917346b8a4370d25d9b1e63316a475ba2ac5ba97cc720dfaa4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 0.8655119260074571, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:24.235550+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3e01915-6b38-4acb-baf7-915af95bdbe0", "result_sha256": "a4cfc616e4c64f8b499827067faf6275747c2606c6edcf873a250ce42662763d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.028883810970001, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:50.903132+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "2839e519-37b3-4d5a-8129-9336cb5aa553", "result_sha256": "7fd9c5016c669ee49cbca163178f859df8f49bf144fff4d854db65d03866dad5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001834, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 1789, "key_errors": null, "latency_sec": 1.2632231330499053, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:49.839562+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "07927b8d-0829-493d-bd8b-6fc51a859edd", "result_sha256": "9dcb0bcd0509a78ec165c294f85a69fa8a786a8eed258e5ae48ac2c5262fda0f", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.8017545890761539, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:53.065247+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "47e81a34-3a73-4720-b516-421c956fe802", "result_sha256": "a72868765f0be7f248f794574d24910bbbc915b16e10c099e515e0882595ad9f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.7448666499694809, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:07.303418+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b88d8ea8-f61d-459c-8062-20024fe2eff1", "result_sha256": "fa9d48bcd2ceac80c53b437d99c4eecfb43b4ad872f3c67ec34c2e96a3d6817f", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 2.162442251923494, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:08:43.118298+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8f907b7-1a2e-4612-859c-f781522ebd71", "result_sha256": "187d75a21cb122c16c4508190fca31b382434386abc3c4ce1a26793be217850d", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 1.1191340850200504, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:51.027332+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "93d55d3a-1082-4de2-ad84-60e757a60bb8", "result_sha256": "dcaf868cea028e8a95b39f5263fce7083a91565d83d438417b1b5484545181a7", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.6890128280501813, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:23.721453+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed8620c8-0858-4f39-b38f-19e0bf1aa632", "result_sha256": "a74ba044a91765e9514b8336216d4688c254df5aad1e4d4ce3809d0e54486a36", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.7187646010424942, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:16.170183+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d58ea2c-8f01-4c50-a725-204f59b00b79", "result_sha256": "de6259e33badcf6d4931276fa701e6223f0f8c79c017cc8e47278327905be972", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 1.287420937907882, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:49.999170+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5a22e007-794e-478b-bf8a-f637fbdc6ff7", "result_sha256": "49d3d51a745433343359bf21aac7e2484319e21831bd1e4d39dfa1fcb201ed6c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.732733684941195, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:38.291300+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "16760126-33e9-4589-8f0f-6f86a6023ccc", "result_sha256": "af44bf85340ed7dd040439cf682bd109fe9c47e2b99d7cc5fb603f8e1e1584fa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.6687635401031002, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:55.293378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6996de0c-e567-4109-a3e0-9396be04cc25", "result_sha256": "dc7a11e3cdb26ecf2210e89100cfe02a457fb39e84afbcb17a4a19d4ed91700f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.6955347320763394, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:02.843509+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff4b594d-f548-40a4-b826-326947dfd8cc", "result_sha256": "e6dccb276d75f2e488489b9172911cfaeb85bb21cdb44effd3637344b553e991", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001859, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.7073399859946221, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:24.544778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d030b729-e4c3-487d-b97f-4970a0f9f889", "result_sha256": "a8ac0954f9b44451092c97faeccd08dbe6ada7492302261b82b1faa2e5151186", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 1814, "key_errors": null, "latency_sec": 0.7284363619983196, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:15.049767+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c36cc3e-7edc-4296-b1d0-f5bb7eb36f84", "result_sha256": "5129b0b92abcca97f4357f97b874fdd43939c9aa6f6d35de7523ecf9fc7087d6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6296606309479102, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:41.441504+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "68402b91-3e41-4180-ae37-89c994e8313c", "result_sha256": "135f150c45ba3e293100843b04a5e0d902e3f2f16680168a3a412b05c786eedf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.5867185749812052, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:12.923730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "90fba422-7689-49bc-bb93-c7ec449fa392", "result_sha256": "28978188d65e924e7633e4e28672738c0b5070e75a7fc179038fa8065da40f98", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6663566989591345, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:26.328183+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3aa0855-34e1-4894-89e0-f7fe4889710e", "result_sha256": "15b3627c05771153820d96d426cc386be38d6f47cc81ae9e5fb53b3427e49ed9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 1.2270294670015574, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:08.928489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "0ace3fcc-3129-467b-81ce-f13e57870fdb", "result_sha256": "c1cb2dee40915cef2f6a8ce472541e1278cf881c03140efcf29042278ee35561", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 1.1537323299562559, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:05.295056+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3b5c83d-b5ab-4ad0-a209-ed4d7f456bb7", "result_sha256": "91bcd43582958730d8024d0d41b2e393cc2fa1503e58191e96d3c35077b0f534", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 1.160009917919524, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:43.117610+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a7d654fa-78f6-4b41-ad82-add4751f6503", "result_sha256": "9ef740eee542f9952dfb1d4634f8333f0cab8513e483e232046a41b1f18ca5b9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6489058720180765, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:14.821221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1592fc0d-a204-4eb0-89eb-c7443d82b82f", "result_sha256": "dd212e308dbf0942c15323b2c59ee9ee3d9fc5e68936a471662c67a1d1fe701b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6487378149759024, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:28.083656+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e116b981-8897-469d-880f-0ff77b1258ac", "result_sha256": "8841d5c5794c5a084827a3834778bb38daa8cf6476e0e7366f2a7e6a35f84911", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6295738230692223, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:42.093611+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a16ba48-5e53-427e-9cb7-6dbdceb7f7d9", "result_sha256": "d092978eda017ecf25bf4848608a986d423d2dc323b50dfa62c9bd122b2ab923", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.7493067380273715, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:21.051287+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9e72e0ed-ab6b-4cdd-90bf-277d3e5ae340", "result_sha256": "0f86ba574fa36858f09ff373219dc92b9e02bfc37d07978dd4a489fbde6e524c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.6820524079957977, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:43.990551+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1cf9f7a1-1ceb-4886-a912-75ab5af658bb", "result_sha256": "5d1c2280ab251bcf22ff69ca0133f329a2f1f8f82233d5872c158054b40d4d4b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001858, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 1813, "key_errors": null, "latency_sec": 0.708524595014751, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:41.859142+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b10bb794-4577-4870-a49d-f3164c1556b9", "result_sha256": "5be0d30beedaa6111449f98948b1ef7e7f4887dcf8bfa0ed976252dbefb9889b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.7490804670378566, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:59.598089+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b0447cba-406a-48ef-b475-281e62624ea8", "result_sha256": "5b1648e2ce5bbd706cee932b988416826140bb1d444bb5a1f32bca4f314217d7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.7456695429282263, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:23.485506+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "27461f44-e26a-44a7-a1a9-804d3256fb6a", "result_sha256": "60a8fe029f88eaea22eafd9a76f698528f0c80213ecc1828e4c758df596d78cd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.7122178559657186, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:20.609225+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1af16c4d-7085-48f2-b93e-6ae8dbe49a82", "result_sha256": "d76dbbbe95867fe7db36b015d1206bf56f50a0d11f994dc1900571f6e4aa718a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001871, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.692226946935989, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:37.053278+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6549ca3c-8e11-4d14-b45a-e0c7c706d679", "result_sha256": "b2ea7cc46273298a3b1e8340b1ef6069c65d0503434ea44a1c16726739b77d9f", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.6227974928915501, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:44.718507+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "c8d76b7b-d476-4d89-aba2-caf0e3f41e7f", "result_sha256": "11f314723e1ded1408f847d5f4f7f261f031df2d793379e323906159adb66107", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001871, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.8302260030759498, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:07.769551+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d1de3eb0-3a1e-4c44-9c6e-455cb9f0f1d2", "result_sha256": "9285813d5740bb57eaca06bad710d08d5a10deeec86df7d37da4f8d2f9312717", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.5499372149351984, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:41.253056+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "353f0d0e-9561-470a-bb02-8eea8fa70196", "result_sha256": "6a4a364dd42df3b9cec9e0e13161f23e7062466ef2841b26bab3ba13b8443431", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.7783859580522403, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:52.104622+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0225c89-a638-40c0-a599-8a4063aabc15", "result_sha256": "407d88dd3b8f5b0a7c65a69bb05bb81fdf020a31c80ca9e5a8d02b3f87d65d3c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 1.1290975629817694, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:35.724880+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ec8b158f-34ac-4d8d-b3e6-289b1fc616db", "result_sha256": "01cb0e7500a0e3d609435abb94b9c9002565345f1090db024ecd580e3069967f", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 0.7212790919002146, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:39.239651+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d3b4c0ed-604a-4774-afd6-2c867621ea3a", "result_sha256": "e3f71745d385c15b09205bd5fb09ef76049494c10e5a45f7fd0895f216c7597a", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 1.3221060330979526, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:16.449938+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f4653d73-7cc0-43e3-ab2f-08b91bdb982d", "result_sha256": "2119efc297f906daaf56b8a6448fb5cbefbcdd9cb1cd8a3299a5a2017b771992", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001876, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 1831, "key_errors": null, "latency_sec": 1.224790649022907, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:32.243852+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e7b8b961-f114-4537-bf67-1c5e79f8e53f", "result_sha256": "3b87cc2b6bfcc431792a5de804475332c3530cc69c7856ee7d39a69371d6a00f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.4663895219564438, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:46.303526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d843c45-0e96-47b7-8564-53fcbb3f1ae0", "result_sha256": "7ffa460e78e22ab91692aa1a2734ef42702bc404566ecedf75194d47bae3c228", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.746270367060788, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:59.594610+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "313f54ac-48b3-4d21-8f2a-ae84662cd395", "result_sha256": "cf5b594cb8d4853444848d5b72218a7791ef93f052ba9f9eee75f8d49818cdfe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.5528238900005817, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:20.154539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "04a856bb-144a-4179-b9b3-3b1459b4a185", "result_sha256": "21ffdb6f9e56cdc24c115b8e5648865ae090e06f05d646e8fd9a3066279c9a1f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.401790733099915, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:53.199055+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "15e2565c-cebe-4856-bafd-2749b14c23a2", "result_sha256": "651eee376ad3957dd1ce55658a82ee839e16e9d2d023f6e7c49370be4103dd6c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.9734950149431825, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:45.311160+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "358035d2-57b3-4565-b75f-1ee0abac6dd9", "result_sha256": "6166456f1d154acb53b2c2e30fce573b0099575c2de32c8397ad19736228ab36", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.605119596933946, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:19.127690+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5f4a0c06-05ec-45a3-8f9f-ae85e211443f", "result_sha256": "fe4b21a3a25cf898f8740fe2d3e2bfa3d2f78b5866a8c1bb3d0448b7c7afe350", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6998839159496129, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:25.193720+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "c76671e5-47d9-4320-9ac4-660c46810308", "result_sha256": "a9d5f02c5458870f48d6e48d6e4954524753682e9a3c75281e9b651724773528", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6794085720321164, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:43.943264+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "99d8f487-1ac1-4e81-8c3f-81772505095c", "result_sha256": "d343ef44d5ffcf0dfe97577d9408069f1415d5a73e0fbf159214cd7c88a1254b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6840606479672715, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:07.918412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5781b2a8-6b26-4337-99b8-84d75b2039f2", "result_sha256": "124b9243ec626efcd3dd7e65d9e71bd9ee7dabbc05a0599ff4ded7fddc743cb1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6711997829843313, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:11.350181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed39af58-e616-4c44-85c1-bbe8b1c8cb49", "result_sha256": "10f879f0924b65588365e6270f73f2d0b2999a653527da395705108e38811b5f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.1921448420034721, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:32.414291+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "80a9d259-09dd-40be-8619-bcb1866cb01a", "result_sha256": "975c89edd725d3c9fec9d24065031702f184a31b00badc2ebf11bd571e4b9d29", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.608709970023483, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:54.588436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "10f32b68-3ab9-45de-89cc-045b087ea5b7", "result_sha256": "512816103648779cce540274be5f58aea888ebb2e13f0026e4145bfe36544d41", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.191670508007519, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:16.448470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "dd369286-c844-47cb-8251-c8bf4423789f", "result_sha256": "f06fc433d7beab19fa027aea65117819210d105c637ef26e0fb8720fc0c52b30", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.1159568969160318, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:31.948190+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ee2468e5-0249-42fb-bf27-b7c45c116b3f", "result_sha256": "d8c4cbc61d9031f6850f135abfb5b0734a090fd3db64ee932882a2366eae176b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6547806380549446, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:38.263225+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9e3ebc9-0d80-4bf5-8ed0-5137058d6fd9", "result_sha256": "6fa1d15fb1595b0123a822ee323015086a4a61da58fbc2ae8c150b153fce7dac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6877396159106866, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:31.641835+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1d41caef-f545-4d79-b06e-4d3bb129cf37", "result_sha256": "a4422620d7b00f1b4610b82f8c9b66e3bcd9fabbb9e1cc6426234a7b65c36b79", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.2050374679965898, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:10.857310+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "44e829d1-17c8-4fda-8e66-294cbb103e15", "result_sha256": "a0c28846769fbe82653f98026e2282e2ff907664154ce348bd46626b98bf7344", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6711723370244727, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:28.181444+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f7cbcdf0-2e3e-4851-b8a0-a02f62609fa2", "result_sha256": "f265bc90791d8c6f4dcfe9b040935b481ec3212cce57f2e560482c7ce53536e5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.9060772069497034, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:52.407512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a1b7acd1-a18b-4374-a704-3731dcd9b405", "result_sha256": "8805b9ded0574db69e2b90288960c9f2c9f14aaf8113261729831c4d30a946bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6025357840117067, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:44.823594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "7e8f5959-ad3d-4f7a-a49c-9f215dbf53dd", "result_sha256": "b4184c5cd30a5df9045e52a862ac4df1cf0c1221d2c23afa9431facd6601414f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.696687203948386, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:27.803917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4b12539-f76c-455d-80dc-a9e78fbd2110", "result_sha256": "281e700a3da09adbec4bf2b34ab82dab4efa7245af124edfeed383b6724fb576", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.012361787026748, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:04.332976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "0fbd1f81-323e-4c6a-a761-08939e609d40", "result_sha256": "a81955f4a492f1a3a5b9331b3e824e70292bd7d05bc447f7534d7cc98f1b825f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.25267909897957, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:36.837927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "369f985e-5e10-4bbd-b739-c2dd9a83ed6b", "result_sha256": "a528e672c85df358508304ab1472959695dda9b355d4ca933aa2dfb7f29d8006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.237291747936979, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:05.451520+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "3d86bea7-8314-46e5-9d66-458eecb61e01", "result_sha256": "8c09a788005e2553c9a664182308c9a441136e97955d36904baaba4e889a8780", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.646148188970983, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:19.864552+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "04ea5be0-187a-438f-ac4c-85d29ff0fefd", "result_sha256": "6d209bbd889c518aa013da3d74067c624a5d68d32a2da68d5069dff6156d6935", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.4985559859778732, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:34.822755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "dc2f9788-f82e-4825-ae25-1a7fdb15bf90", "result_sha256": "4e01a069c661ae690305fa644fd26de94b73be4584e446defaf45f6c9188828c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6488368820864707, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:26.955521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e761fe2b-207d-4c0e-b95c-bb1fb819ef79", "result_sha256": "6493795fd496657fd5cb5a67ade6bbd31d168fed35cbc7e9535db694493fafdd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.1579677530098706, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:19.486395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "639830f3-8319-4674-bbb4-2eb857c3feab", "result_sha256": "4aa7ba055448b60b40ef64369f35a4af288851c4c3fe46577e5a83be9abc8101", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6923633040860295, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:01.700512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c71c4c7-daf6-4980-b0e2-e9a24c4f1f90", "result_sha256": "6fec36116a3ec8c020a900daa758911b07c39730ca9d98381d48efcba01a2034", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.1643199749523774, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:02.136412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "dd5907a5-b441-48e7-b466-e42f56ce36ad", "result_sha256": "eb33b6ef16fcd92f322d579030a9b768661d195598b5f6d71d8081a7dff2b76f", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.1988353540655226, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:33.445913+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a5febf66-1d93-44de-a9eb-bd6beee97fd1", "result_sha256": "42104069c0774373deace678546443b172bd4e65c4a4b2e8e22a8adc3baf0b03", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.4818869809387252, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:22.902084+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e9cc795-cb9a-4f50-909d-ffafe981bceb", "result_sha256": "2eab14719011b35b0e1e4fd97b87dd852100bec341bb52bb0dcc576eda22f12f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.5262800980126485, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:00.909745+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "801c7d71-2e5d-4cb8-ac41-90e6d5a88d3f", "result_sha256": "86f82d714e35f0206350bd0ea1d770117a54b2a955251a1301ee63d777ec94ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6521298569859937, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:27.148458+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1617b36f-7bf7-4d35-8865-f2be591283ac", "result_sha256": "b8b457b31d3e325b1c6f607966565f9a52cebd6d9d8307f695eb39ed4dc316dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.2166944530326873, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:15.728186+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "66387cef-1453-4903-b254-b1fb734bf920", "result_sha256": "9326eaf2917129beff85df7baf63d9ab72230766a268a4eb5e6160648bb2d8a5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6845902169588953, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:42.691350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5f57886c-3237-4780-b118-53bb101b1b83", "result_sha256": "9aec025a440f0caf9e30f9046012f08db6f9a3883a653cd8d481dd9d901c5517", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.2146584800211713, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:06.492022+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "637d702b-8094-40dc-9657-75da5a2b8401", "result_sha256": "99393f6b05d54d820925f41023cf7c9157eb6a7e7e5290e8f81dbd1a72f40840", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.5298772959504277, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:30.243611+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "965a6614-92b1-493b-9c3c-3bcfc28918c6", "result_sha256": "593ae11c95d87ed4040c67ad62420f367119b396df155ce3e41e2164afacf37b", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.0590460241073743, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:34.909059+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "8878a0b3-bc11-4607-ab3f-6b9527c111b6", "result_sha256": "e75d8d87bdff5bac30b13bd5bf8a9f1daa1ed66cabc9b5222782a3f1216aaaac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.1331924910191447, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:52.032496+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "54a2821b-9c1c-4acd-9d36-07df17dcd9ba", "result_sha256": "d0a7e027699a10c69d16371e7f9aec62b5459a855c5ea599715c3c5446d43b03", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.5552927440730855, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:20.832937+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b900874c-8a5a-4c46-8423-fd937ebb9a97", "result_sha256": "e413789214b3632521304d84068078dec3a221562c8e6dbe04aa29e046ddbea6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.7205281379865482, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:29.147676+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b695afc-8227-4dbb-a43a-f6efc828f335", "result_sha256": "a8bebf45af99d2d76915f47fbb30952bfc3087e260f71dbe5af90fa06b46ec81", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.1657958870055154, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:08.613167+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "17d0d83e-fe79-4c69-8d62-513afe132598", "result_sha256": "82c308cabb88a8b5c5cd1a4403b1d20b6ac86c221f0328fb5b8fa09f34d8a61b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.2484773509204388, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:10.896883+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d0c2bcb-1287-4283-b031-3bb836babbb6", "result_sha256": "dcc7a6a6d0d3a6981e379dbd8be7170fb36b542c6f10b17a55cf56617a2a1dfb", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6799873120617121, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:44.710854+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e198f3d8-c3fa-4ab5-98ea-f4bae1a0e018", "result_sha256": "a28ec120571bc182402d668f10fb307af56e67784c808ef7e045b77e110d7b65", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6939985209610313, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:56.578230+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6fd999fd-1f57-4045-aa34-ce685da1c1b3", "result_sha256": "77febb7e134ecc5389b892e0bcc1d3258abcf111ff2e992f54a03ac5aee11faa", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 0.6931344530312344, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:25.208418+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "11d9618e-e14c-4cf4-b372-4210d14c2590", "result_sha256": "6d64b7bfb0b761fd62ec114b417610330a35c8c42fa8749268ffdaa2e6c0da0c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001866, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 1821, "key_errors": null, "latency_sec": 1.0267259620595723, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:47.707661+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "3d5c837f-6595-4e38-996a-7d9dc3e4ae27", "result_sha256": "29bec49b2bc0a5c5f7eb7f29cf8047a679dbcc0775799f4d1fdc583833ada0a2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.503313688095659, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:55.089760+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "bcda5488-d6ab-4ef0-b4da-b7a53b32c1ad", "result_sha256": "c057d3c00d2fcc00458fb2a41dfae1e3e7e1462e8773138181a4d4c9b6fa4b60", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.7154779200209305, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:16.740346+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "388469e9-763a-46e9-8724-a980099cfcc8", "result_sha256": "c637ede3266f9ab0c0d1ac3203ae2c1cac4ac09800af80872f118705e1842d1c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.8236723990412429, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:09.840486+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b066fac8-4087-4e54-a17e-661ba2715df3", "result_sha256": "5a485ba09a4f7d47a0b07f0fcbf489dcc4836922029837ba96cde82fad40f92b", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.9188094690907747, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:08:45.078487+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9dfcdf46-58b7-4796-b10f-bbe1297a048e", "result_sha256": "b8e836b49b0e70086bc22c8c2dd63d4a55491555f84a1e7b0198a8b375058712", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.8520750760799274, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:25.520065+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "4ee403f2-55da-4fac-aa98-537ebdb27e26", "result_sha256": "7ba7955357d8977612a4bc3a586582655a7237538d8a41d76febad01418a1341", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001879, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 1.4299144460819662, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "B" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:10:03.010920+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1483f37-27a6-468c-af87-76f01083a541", "result_sha256": "b8a5b610c648bc9ddbc4e03612ec3d19b5bad9762f46565b480f20f912acfd51", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001879, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.7708842800930142, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 8, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:12.065413+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9fd6a58c-d333-4a6d-abb1-8bf357a7bcb0", "result_sha256": "61a7175a025a55f4a0d979e406078c82747415ba6682aa27171a5aa80b16d3e3", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 1.497398411971517, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:42.510648+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9060ba57-7333-4c8d-ad18-144aad386ab1", "result_sha256": "c42e6bcc93ce4a5b714c47dd5cfd1b5cb777c816532f1e1ed08d9822cd97b67b", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.7174117099493742, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:35.866300+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf0f76a2-1794-4ae7-9d80-0c7f8990d6da", "result_sha256": "e52537a181e97283f01d08b480d11b9e24fd0487304df8d2596863d30f0b072e", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.5449982489226386, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:45.182474+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f532c2b4-12ba-48a7-af2a-311c2ed33b3c", "result_sha256": "396b9767ae1cf693948920d143b6b2399e2b396c07e65ca6affa9e40d8a07f79", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 1.2464016140438616, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:31.369785+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a3c4d2aa-b168-44a7-8d0e-96030583cca1", "result_sha256": "82d67e18d3e354c8971aff18099aff4129d592246096f03e8e04b57a58bda1f3", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001884, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 1839, "key_errors": null, "latency_sec": 0.7445697099901736, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:57.918891+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "60160b95-e84d-41ef-add1-74453625509b", "result_sha256": "9eb4570baabce7e9dca6911fd9b8087d4a89fafd91b4a69992ae9a6da3f141a1", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.5488558019278571, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:17.864952+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "28836c1d-c602-48b4-ba1c-47fce038beda", "result_sha256": "8f0e52cfa2477fda90e63c1ef4c960c916e6d8012e67ac5a3587093abf94ca9e", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.7649832780007273, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:55.317079+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f12be6a9-7d69-4bfd-9a36-990d17c7d936", "result_sha256": "7bf9fd959f4e3d9e9ed2b71de20e53f940374ec7e68693e764f7c02f51030355", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 1.2831723470008, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:12:45.578924+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5ea67467-802e-4b52-a1f4-14f1c0fb7d6f", "result_sha256": "c8657544bb407da368029a863dc01d2a9dbf77653ea46502c6982ee79a73c088", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.6376529389526695, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:55.226416+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "bad0eae4-d72b-4af1-a03a-9bf6fce94823", "result_sha256": "3135ac7353d14e5d6cd4bd1b80e2e8f7c09db8db3ada01db64c1a91f36013ddb", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.8963161070132628, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:23:49.218944+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5cbb7df0-7561-456e-ab75-40300124f263", "result_sha256": "1187e65dd110822e91f0a874becedae6248b7928855f36098752e74524b84430", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.7774441180517897, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:36.519461+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f6b66a20-aed5-4318-8465-2b079a2c3284", "result_sha256": "b7f541b8bbc0dd6803a9c4ed90bb8ea34c3e40e236ef5940d72f14549ff0ad1b", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 1.6219840409466997, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:12.913944+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "046fcb79-392e-4b6a-9b51-b431c9042c52", "result_sha256": "127ce34ad07d390980735f4728c2176e39642c75d9aade87bdc2c18d3b54a974", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 1.3117674749810249, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:14.065711+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9470aa6-2ec8-407e-8de7-09775db65673", "result_sha256": "88d8ff93e061e96f04f94b9bb4064ce0b4d2eb9ce7f35a46aa38349b6212b01c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.6956140589900315, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "D" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:09:19.505964+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8c2e4d3-1059-4d70-afba-a8c7af98158e", "result_sha256": "b19ea592ec2248c325427f93b6f4b787d23a1f3a1277590fe6f6224efb7629af", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.6635722590144724, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:52.413282+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e15d0845-2ca4-4e61-8fb4-7dc09fb90060", "result_sha256": "f938f6515efb3f7ce2a4a4058d586b87092f6729d1a9f58394db2dc4f7947801", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.6826033380348235, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:01.607176+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "88bd0c5f-508a-4f5c-8572-1458c075f521", "result_sha256": "251602083a61a03deff165fea1a10f5c958ddcb9681bf32ca089c154cb17fe6c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.5572165860794485, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:10:00.951268+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "cbf727b9-f5b1-451c-a361-744a432b2f2f", "result_sha256": "646123fe0e59903ac5a8560cd3779dbe6dade1c54f4be4ce7bf40d722db8f8e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.6131517040776089, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:04.505266+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "fa4d5115-2faf-4088-82c0-04e6a42365ac", "result_sha256": "fdfd7ea09f3d14edaa9e4d22268b39c97405e3bb53ca71fb0175bb41a3ff8c3c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 1.2942894760053605, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:06.461918+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "af8910c8-8029-46fc-82cf-67b87ba2973b", "result_sha256": "aa56bae31865ec36a23f1e39cd652f025be1f21b7ccdcd3c534769fb60ea94f7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 1.1017295399215072, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:36.473105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e5432e5-e2c4-41fd-b734-22ce3dd6b405", "result_sha256": "69f8b745aa06a20476dfd39cc4d876c14c84e4a1e55cce764ac45c852e8eba0f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001887, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 1842, "key_errors": null, "latency_sec": 0.7621962439734489, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:56.450597+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f84ac9e4-46e8-4cee-a8dd-b6018e80cef5", "result_sha256": "6760868fd0c729e37e76bbb54287045d2770d9b3bdbd9cd7ed7edf69c2484c69", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.6871591559611261, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:49.127607+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e63bf02-d764-4a2d-a154-1912745f9b28", "result_sha256": "8de48628448529b504e3b703e8c9d739128059ac28268cb93960323370bc429a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 1.272428939002566, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:29.279850+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "92f1a972-0fac-4a36-896c-d6824f0169d5", "result_sha256": "217242607a065821784554405ebd3dfd66fc9aa914d868c723a8eb0adfc36947", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 1.1421308839926496, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:29.522634+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f187586e-2061-4b2e-aca4-286de6d744aa", "result_sha256": "594163b80df684721f7551eb5fa0fdadd4822f1ad2da18adbfd0562c398c5dc1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 1.132455668062903, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:25.065011+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "88543dbe-dd62-4ca1-957b-0c0ab2beffe3", "result_sha256": "61e6ae11cbaeea4915e5af26cc168ad5d865184ab7f00baa14deb81be413b9be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.7797020140569657, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:45.233319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "49f0fa7d-e1f8-4918-b4e6-7ade4b566386", "result_sha256": "b472a5e0cc5413190e3cdd8e8c5113e9164a557289e12cc0b22bdcd84e9b7db4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.6596188719850034, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:31.486151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "46f12893-2ce3-4f0f-9d0b-42b258bfd0b4", "result_sha256": "474f6f960e0355ec76ac51f42b92b6ff1d086183a2cee0fbf8e7f7a6590e045d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 1.2576991930836812, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:25.130326+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd34d6b5-c9d8-4e96-a812-b23154e4e0b6", "result_sha256": "2888c54c34685c07bae551a45dab2e49baf9a90f3991beafa0b64a2db2aab40e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 7.462108353967778, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:52.639394+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c0cb641-45e7-4dda-8fa4-d6fabec946b1", "result_sha256": "3748fcd74c1afa9eae5a8ff2691fc44bcd95090f85b71cebe3ab6dd03c272511", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.6810645760269836, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:58.003539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "8b74d8d5-3762-4007-8008-7cc746644c76", "result_sha256": "9a5334fd76ffec45468ab9aadc1daaf6cc9fec80b65a69fba22080f2448f304e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 1.1099543180316687, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:08.157314+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac3afbcf-b6b2-49b1-9b9c-660b31157a7a", "result_sha256": "8b4fe49764c3994712af8915cb945de444e615e8cf488ab68da286b42c5633f7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.7692851269384846, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:44.696589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "bc2682ad-cb4b-4d31-b3f4-09c63c6658f1", "result_sha256": "475ddec84c7b756dd6ee811f9b7c8c73d543d48c9d13f350f194727fbe33a4a5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001832, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 1787, "key_errors": null, "latency_sec": 0.7499516110401601, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:55.909842+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "9ab4536f-6fe8-4cda-a28f-2f0be40413c2", "result_sha256": "c8f28a533f4715b7aa5e958d14ec0a3929bd6c0f164937bd6129383b60b2e3e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.1849263879703358, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:47.032432+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "fcaa6703-215f-4fc4-ae95-6ec535acc98d", "result_sha256": "4137be2e894adc98ae28b09959ce6bf7ff0a1a47ae888efa2c9b510961e15fed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6823796370299533, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:11.811147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f1f89530-849b-47e0-b1c8-d82be58bb401", "result_sha256": "5e390253427820033a6894cc0df4249b4494f96a567824890ce7353d8f11bcc0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.2020029140403494, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:00.859773+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "aae25dc0-3ad3-47a1-b113-d2a7e1777ad9", "result_sha256": "f2baaf5404093e31581486290c64c309f4b3acef98170df5c2b6b0b90a549947", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.2078757310518995, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:07.042823+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "28f24581-3182-43e7-92ee-bc7a1a8828ab", "result_sha256": "0df35a939ad52387c91ed560c99d68ee2604f96226b3aea34d4a3bc4ce24ebd8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.6774215989280492, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:19.586642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a9062b57-0d8c-432c-a54d-b15043181b97", "result_sha256": "5db7b54c627da8e1699fe06742a7b098f4d4813104f44abd88d700455ee4ac93", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6728745650034398, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:10.833369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "0f6f62d7-14de-411f-8c70-0750eec3d0b6", "result_sha256": "1de22aeed107ac4d6faed55a1511270cde7949d66383344af06cb1725b2e7f37", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.7056205479893833, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:52.633468+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "57830d2b-591f-49fc-8d7b-b3f4bdbb9dca", "result_sha256": "0c90e42fdf3087ec4a2e0d716d4ee6eec4d040cceec10fe73940ce1e9becb667", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.7625779110239819, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:48.848551+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "031a76ab-4bad-4d56-9e2c-21c784fd892e", "result_sha256": "464ebdc1120c54efd3f46de928f70f227a7194c5b035b55782395d8dcb8d93bd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6311255840118974, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:22.526970+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "51b66df3-f8ea-4348-9e0d-66f99efc4a4a", "result_sha256": "2593160167ce15c5d88fe5d06d6efc1a1f07ac23d06ca0941116b51181698455", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6317133799893782, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:33.085598+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "cda5d1e4-4d4d-4fa5-a6f3-7e6bbe053d6e", "result_sha256": "9b2c0f91aca36a0691f1ebb02e96d6c8cad09de924d7f9223c133e52ef89868c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.5498646770138294, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:49.620097+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5d849b9f-c2c6-4d5d-b8dd-1b7e37618404", "result_sha256": "5dd3f55187d9187fc39cec4ced2e47840b32b35eb34f5b98494159e42fe1277d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6354702030075714, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:34.567063+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "eb3e4fdd-9d60-48e5-b860-5990c4aedf28", "result_sha256": "0e19374ed96a21eaa6cc9351c34f7b42e735cca76d7147da12a135a7fc7dd664", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.7872187339235097, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:53.969648+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6efaa5e6-dc93-4f76-beda-b2aabc6ebaaa", "result_sha256": "edd413cd9db2c0e9988742dad3a969e01f7e86f91e5cca07c8dcb9043816a491", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.0793831530027092, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:35.182436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "cded12e8-28a2-4246-b4ad-727a6326b1a2", "result_sha256": "812868f71b6fd086c6be43836897ce23f8b74641c1b73c5cd4e078c3e904e404", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.9681625210214406, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:37.088210+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9b1c127-d3d5-4c5f-bc2c-f3958c810564", "result_sha256": "f00f93ec140bff3fc9f280bccc2041ec0345c7f6626e720d830092401ff5f41e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.195412194938399, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:16.508993+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "edf44851-64e5-495a-831c-1c894b33efd9", "result_sha256": "79e68c574103724a085fa6d3a7f9ed373215a60538e210adafdf669346562609", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.63056990201585, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:56.394000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "3d4eed3b-a60e-458a-936e-0c1a77f69f39", "result_sha256": "5509531fc8ba48ff4cd99a3d0d0fdc47ef45286323ebbc68f5087d2c072ec517", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.7060911030275747, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:41.268670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f663efdf-c8e4-4d73-8f2c-0762dd4c9283", "result_sha256": "6aa85df20e32c3f570eae7db64cca5f3b1c557fd35ecd7b6a390a078ff46c67d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.9711415260098875, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:54.172198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "17b16d89-6cf6-4d89-ba33-7392bf3d0ed2", "result_sha256": "e87898a26966689bc6ed2636629604e9ffe8340437491f6f63c0d934ac0cafac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.7422940699616447, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:04.218410+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9b4ac78-3b3d-441a-a3af-7bd5b0ceafd5", "result_sha256": "70dda6ea5949f54c731e8564f67d6a7b619664cff03eb55bb5d9476529c2bb2b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.4261613889830187, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:20.206000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "c72b554e-a554-4066-812b-b5ef09ea9be6", "result_sha256": "b88ec4a4230bcb01162ae1978e6d82f062d2e7f73a58f679054cc2a7678ad7fd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.0967257610755041, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:35.374061+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e057fbb-1bf8-464d-a9d7-fa471bc99807", "result_sha256": "6ecfa2d0534209286ddae1331a661a75da3323c06cae077d2ab82f21e8eb28d5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 1.2578327850205824, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:24.806095+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "27b5f89e-7e08-4deb-935d-9e2bd9f0a4a1", "result_sha256": "a6df188c3a31a385f6555a71a94a8575f970b36cf65a9b262533ebf5b562fccd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001855, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1810, "key_errors": null, "latency_sec": 0.6843972780043259, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:22.217454+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "c9ba44b2-b1b4-4417-bf1e-6dee434bbac8", "result_sha256": "11e1e38d568abcf14d18d6c714d008958cc62e8fdd4f78fc9f5301555fc9021d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.7162115890532732, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:35.120014+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "162b7806-7c1d-4ef7-abdd-5c14389cc774", "result_sha256": "0b546b42a250d06977fe0442a0c798a67a3cf962644ab97287f4b8a3bc2a44fa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.803821467095986, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:39.783832+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "46fe6c7e-4f5b-48dc-beec-235407cd62d4", "result_sha256": "c4c3bb720ea0533ad185d2a00043e4e1516923360690b4adb3e0280ed8ff494a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 1.2955318640451878, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:50.396837+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "ddf604be-9e7c-40da-98d8-432eb151e3c1", "result_sha256": "d4c4bd3565bf2e4b01fe81861e32e9140960dbf31235b4aea46c10f688f4d536", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.8769601780222729, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:35.499249+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "672e6819-87e1-46f9-beb9-38dd50fe3d0d", "result_sha256": "192b30fd1d03aaaa927ed99369018a8f250441b7d0171dce53c60b1e1fc7848a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 1.3455338600324467, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:13.356047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "da2b06ad-a579-4559-815c-d431f3c5547d", "result_sha256": "54ede191a48f520c77d45a0bdbffef00769e17996191e325ca59ed936df61cd4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 1.2322405689628795, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:59.341459+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d627be23-ab65-4048-855d-98bcc93f9489", "result_sha256": "cbba9dabcbe7a791939b222b4a18f729bd94d5065d73b0390a2bc907b708f412", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.798267264966853, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:00.342389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e71dd619-66b1-4069-abf7-0003701c8680", "result_sha256": "5882c8d8df01b79db9ad41bc65842420c8643b4be4a61ab5915ff4ef722d3cc4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.6820974289439619, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:16.564212+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "f974f2e8-91f6-4732-af7e-9932c1a71251", "result_sha256": "1226e3eb40d0a5eb136cc216daa36715c5461c7353096d912b03f80fb3cb9f10", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.530803058994934, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:56.550111+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "35132394-279c-4ef2-ba8f-9a42e61be033", "result_sha256": "9783b2d56265486ec25982d9f2cdf63a557f2838ee2192fa3023aaf49a2f4ad7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 1.512369236908853, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:12.753624+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "2bd7adc7-2bc7-4600-a1ea-64dd9ef5de0e", "result_sha256": "cbf300e134ab3bdb47d626a033c73b8058d7fd6e00fc35e9f07bafcc4d132810", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 0.7567058720160276, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:41.506348+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "72ecc1f2-8ba7-4748-9409-68c92a7a089b", "result_sha256": "56a4b150b3a42f440557c4afe50bdc3d65436b4154ccf0ff737290697b3cf9d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001847, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 1802, "key_errors": null, "latency_sec": 1.0591772539773956, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:14.171751+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "acad22fa-8201-4898-bb0d-27ea63d015a5", "result_sha256": "ddba3d25979c03e7a49e43177d4c60e3d940f56d5c1ddd005fc474cc962df49e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.9657453109975904, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:56.661668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "2cbe8dd5-d814-4148-8e5f-67130e8c4288", "result_sha256": "45a5a31a117e8b28fdfcbea19e027aa6827423e00172d22d795115d577999dd4", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.7932344099972397, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:31.111793+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf2b426b-d006-4a17-8b77-bbfff1815c2c", "result_sha256": "8b01e85a049b4b0868a82e962f81b3380eb94a76ab27b2422ff165785722a7b4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.7794589600525796, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:56.815040+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "57116497-5b42-4263-a2db-b638316211c4", "result_sha256": "86e4328214a3ff9fbcadaf10e881aee9f1f0fdabd817773e91be5b022b724b81", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 1.0331482499605045, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:32.980959+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "49365cf0-8e3d-4e48-8114-2f9a292e1028", "result_sha256": "b2bbc71b41992e5796d99cc0aa455392580ce738d744a603639b79d599e4ea00", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 1.2426200719783083, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:41.463580+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "a3e59f62-da96-48b1-8a1f-e5dbc8acdfe2", "result_sha256": "4c47048608d86fd222f20eaa729b69262d1275d7cc210483c08aaa5154c42235", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.6509498879313469, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:38.333998+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b3a4cc9-08c1-4594-a9cc-1e40e3a53eb1", "result_sha256": "97ae0af2f7f1a393a5670320d256cacdbed0d4b59ce2748e7792133c955051db", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.6525632579578087, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:50.728492+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c8f7c0a-495e-48bb-b91c-8a45efab4a69", "result_sha256": "477791e9fda183ab2a0d63ed066b75533dc2fae40da68e6a8a2bcafecd7a992d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.6650176219409332, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:33.613822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "00e43743-f4c9-4b13-810a-f36e440670ef", "result_sha256": "431ce04eb2fd40ae88259d21a62a0eca5c52f313fc77c04c0c981b83a09d4e7d", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.6845004160422832, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:01.558564+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "305dd185-67fa-46d8-a19a-e5cee5c5828a", "result_sha256": "74232fd2557bab1668cbdbbd348350d949b3f2e8ebdfff00b76a387fe54b854b", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.6990754149155691, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:53.803427+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c25f603-82f8-4acc-a5e5-84da1a47df5f", "result_sha256": "4b82ee3b6fa80314e220b8b18c557fc1bd1c2f0e1530259e90928f5e7caf9a3e", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 0.9393656679894775, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:40.380993+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "6eb00371-6699-493b-aa14-5d9dbc4bc286", "result_sha256": "6d2d7e3a341b88f8536f27861fcb65d07a91adc070e1c4eaba964b4c832af483", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.001854, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 1809, "key_errors": null, "latency_sec": 1.0365102919749916, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:45.836872+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3177125-fb63-4ad2-865b-55078d7cb5b1", "result_sha256": "24cdffab903500877be75957c8f02ef94e1ffafbffe25d9515eae5c0cee7d8fa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6092378080356866, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:04.433946+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "60abc20b-a4ec-466f-9c5a-14d190f8dd96", "result_sha256": "ce9bbbad53edb9d347c4a57b8ccebf5dc166d353ae3f15be8f77f251dbe9fc53", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6923429069574922, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:30.827743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a856ecd-69fc-4658-90fd-b84089182467", "result_sha256": "909e569aa776bad647a259b7d290a29cb8a6e98585dfd8963efcf3bc21265b19", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.5576154210139066, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:48.335943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9d5a18a-6102-4ffb-958b-772719dc3d5a", "result_sha256": "55242bd30e96d739aef4b574d5a949e82757c7034158fc7ed283f3a84d6228e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 1.2017014189623296, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:26.049509+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "e3aef8df-d76a-4964-aa85-323c158b70c2", "result_sha256": "eaf37686f104e650e42783ccb3b21df11f80d5f4d652aa6f874e06a45cbf7d83", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.5200565040577203, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:01.150904+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "98a70ca2-9d7c-484e-8dba-b3512e1b52ef", "result_sha256": "499aeaaf9f896ba29ed5c209091123098b3e880c82537460caa42cbc79256385", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6902334289625287, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:49.146740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "cc088b6f-ecb5-471f-bccb-1536c01ee73a", "result_sha256": "899b64b58213d989e051553abd5a769d775d61e43bbc65a97e8e67819ebedb18", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6572984280064702, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:16.758020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "020bcbbe-a4cc-4bee-a786-3efe7522f22b", "result_sha256": "194f423f37fb354df8c2601244e026f6c99e59db739328e2711ddcd785d4ff7f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 1.2315073180943727, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:04.853700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "bed17221-e443-4481-8fc7-30227aba4ac8", "result_sha256": "a53a9f28783dd76b9fa17fa8dc8228dafacf2354e3e38270e833a597d44b04fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.8103386390721425, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:47.585198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "b9d5c079-f75f-4cd4-99a2-1390359f69a3", "result_sha256": "c0fbc02e3ad550bad06579e4d668b90d56d6a3ddf7903adb302b546ab00b30f3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6499043039511889, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:47.444925+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "026f4517-74eb-4979-985e-2b2a67e8f3d6", "result_sha256": "73754bad1d81722d9bab0f362fc6aa0b0948f25498be1390c9434f285c4fac5c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 1.2700057619949803, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:53.660233+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-haiku-4-5-20251001", "status": "final", "included_in_comparison": true, "source_attempt_id": "5bfde5e3-5965-4222-911f-1d7171b72f89", "result_sha256": "df17e46e73789b597f7439232738ac810f92798172a7f7f1f9755c43d3b960a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001831, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 1786, "key_errors": null, "latency_sec": 0.6765057180309668, "model_name": "claude-haiku-4-5-20251001", "output_tokens": 9, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:40.615535+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "52f53fa4-2e1a-431f-8ba5-93024aed194b", "result_sha256": "03fccdfaaa83b7717f71d9df62fc2badd85963bd53b9e152332654a727f09cbf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.25693367398344, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:17.461672+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f167cf7a-f107-4d9b-bd0e-c141092575a0", "result_sha256": "301f5213bd9a486bf0e2693d2660aea8fd7c6d0984d51640be1f15fc43a2349d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 3.634553916985169, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:15.401546+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0c062a0f-3ee4-4156-90ce-3c606ef8866c", "result_sha256": "8d5133b0469b3c4cfd8f3e5a2b46fcaad696b3cb898f5df54874bf2a8e63b57f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.171506985090673, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:08.075083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b538b77-9386-45f2-9d21-2e9088f9a6cc", "result_sha256": "151cbe38c3a85de7d928068a408817d8556c7a7555dbda6ffacdcb3984ccb88d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.7935756780207157, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:11.260023+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b1a710b8-ee5b-4bb1-b35e-f2362d26a3a7", "result_sha256": "ff037c4cba27be623a1786b065352dbf580355e5c3301d0cc9fb73b5c110a015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01892, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 4.038402513950132, "model_name": "claude-opus-5", "output_tokens": 192, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:38.364026+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "64d134fe-c76e-4c67-a354-3f8bcd66f494", "result_sha256": "e19a9eaf780b0c92d2dff9124ec6e5d9bb832bca54762125bddd2e4be045d861", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015595, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.767409566906281, "model_name": "claude-opus-5", "output_tokens": 59, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:50.849037+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b0d493a5-b003-4a10-91e7-f74cadbef85b", "result_sha256": "f2239feb291e6c04b080c0cad23cc6b2ef7522fee6b7380fc31d51606ac761ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01767, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 3.382683486910537, "model_name": "claude-opus-5", "output_tokens": 142, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:40.170784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "cfa57141-104c-4b8f-b026-e39e55d03dcf", "result_sha256": "63804616377474451182e1f11cc04c428c7ab8d5c78e25f798ad02dabe01cd4b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01652, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 3.251068320008926, "model_name": "claude-opus-5", "output_tokens": 96, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:21.688255+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5cb99078-0364-41aa-b79c-d4c984f5f7af", "result_sha256": "6d6d4a29467ff366eba7d2db92e05c1638d2e2c0e16a5d6abf29defbcfa239fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.862789642997086, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:28.328902+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa50f1d4-1e3e-41a5-839c-80f92fb9d758", "result_sha256": "a5fb432fade73acfab7cbd9c2d7999203b1b3b3f19d9adc8e57e216787df6a46", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.0600382179254666, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:51.525446+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "79805876-4f6e-4381-a17b-84493cab3e53", "result_sha256": "a4a1b7a266cacf173df1323b675d8d1500b719e3fa90ba0c57b9438d4b7efa53", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01557, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.7952971711056307, "model_name": "claude-opus-5", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:28.777109+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "79720277-b82a-4ce6-8059-7636fe540fa8", "result_sha256": "7f9765f1e1d1b48cf8d8a1eb4b7e475d44d122e55ba91ae7f42e7de6e901e801", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014395, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.031567556085065, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:49.671083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a7bacbb7-e9e4-4882-b91a-d514e2a7957c", "result_sha256": "fbf6efc023ef9e75504e285c9c074eedb760e4df045e135f7737aa6147e594da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016475, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.6135436730692163, "model_name": "claude-opus-5", "output_tokens": 69, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:55.669798+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "beba0026-d431-4ed8-9b61-c1d762803f69", "result_sha256": "236384fed4fff198c2c1be291ecf177a1f7f44c04ae55a65a572ebfca6ca3230", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.0260190500412136, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:59.920273+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "05e36af1-4e67-403f-8fc9-47fc2f03f0fa", "result_sha256": "70f4b4cdc0585c2ac5207cb3a89241914b9b03b138d531e363e3a9bef789f39b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018175, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.8496161659713835, "model_name": "claude-opus-5", "output_tokens": 137, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:01.090106+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "618311fa-5681-4a04-ae67-3b6fa3b4684b", "result_sha256": "5192256b12f17ddcd55fff4377f4a494ff46dbc6dc3c9d42851d6e282abaccfd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.9901694020954892, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:04.618246+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f71adcfd-afb7-4636-9e5c-165af38ea9b5", "result_sha256": "d0d00266e49fe089a41673013a281e2f70434ad1480df32f286a5a257c9226f1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017575, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.874458491918631, "model_name": "claude-opus-5", "output_tokens": 113, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:05.276937+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d897f535-9787-4024-83ae-aed6fb6a1146", "result_sha256": "debc76c53683e815bb7d25324d910cdc44a1b25c23eadb60863883d738894e1c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018125, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.970839538029395, "model_name": "claude-opus-5", "output_tokens": 135, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:11.453048+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "33eefefd-1e52-491c-b34f-ef28dd67a759", "result_sha256": "f153a4e4135674cf26ea1f9e9b9fc089754b970038638c61461002246db688f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01745, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.3685999719891697, "model_name": "claude-opus-5", "output_tokens": 108, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:43.050617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8772856-610c-4268-8da1-d7a5f8737345", "result_sha256": "7111a8f7c0f8d67116a2ea85e076ff02e602cb210bdb0c260894befa5e18d21f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.8608528899494559, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:40.637959+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e7e602f-c3e5-4f88-b2f6-21d5ce82666a", "result_sha256": "50fd72710dfe8eb72c93cc410deb79ab6e159252a9f55dd5ec6bb70e8879b53a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.3569706969428807, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:10:01.369699+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2dfaed44-5825-4d41-a3ad-e869c22e7557", "result_sha256": "c1f024cd83adebbbbf511eb448d45dfbba4ca44357ccd399c9af7e05a0b5dff2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01765, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.0384609249886125, "model_name": "claude-opus-5", "output_tokens": 116, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:22.307692+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8f3c4208-c22b-4a69-8bec-4e694d1e6e7e", "result_sha256": "f4567d42727bcfeca08a96f8bfccb5fbf8160eda01bf85c2fa82f5bdb90638b7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.0277729900553823, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:06.202801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f78f8a4-fe52-4773-93c6-0a692bad9f26", "result_sha256": "cbf7677b866cebd3b8321acfbd0b13053a64abc3e712282645f742f6d89dda61", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017475, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.466560887056403, "model_name": "claude-opus-5", "output_tokens": 109, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:50.537157+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3f13bf44-7e89-4858-b533-f9b64641138b", "result_sha256": "df2dc6787a815515ac19886bb654d6cdabc060b624061c51f971a7544924879d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016175, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.7886356649687514, "model_name": "claude-opus-5", "output_tokens": 57, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:48.378055+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8b75b53d-a02b-4d31-a641-7c48ee76c658", "result_sha256": "875c4f608290f54be2340de87a82e17142fa56cffad673041d21b43d34e4a996", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.9849438449600711, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:01.153135+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd26c8da-6aff-45fb-908c-15657b7290bb", "result_sha256": "daa04c5a064009f6dffc877d396f14fae2fff8ba33d451d0c13edec887efbbfc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.6283880199771374, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:56.910944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c0ab56ba-c402-4c01-b2f5-1095972e948d", "result_sha256": "1d1dd7e7de63c71b95f02a9208cfea2e883d94d40c4f609aaae20857fb500fe1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.032725, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 12.566488765995018, "model_name": "claude-opus-5", "output_tokens": 719, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:53.308642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f11fa3bd-690c-43f9-b2c8-d7f0dfc8c539", "result_sha256": "275438f8dc8084cc38a56896a17553838ce729ca660e844b3256443ca7e9ad10", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.9293416609289125, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:38.600671+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0c27fae2-8aa7-49fe-99db-d25e69e54685", "result_sha256": "b37654c5d10e760ea5853b66e8afd3049561f88e6f731b286ff9e90725dcc557", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 3.699882964952849, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:10.802784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9ce71385-1dd7-47c9-903c-2d584747562f", "result_sha256": "211f3159a15f639c6043569b86df9c3ad03383e5fed3881abb81fc771e6d3c3d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.783692136988975, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:27.042374+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0d29dac3-892a-4caa-a110-4e68d2a8ca44", "result_sha256": "06cec61f1f549356923cfd5a17f70d5c7ce2523a70ffbfb6f0edbb0ef40c9051", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 2.40151453204453, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:23.039650+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e5d23baa-24fd-4482-83aa-8bcb6e76a568", "result_sha256": "c14f0423c16bc6a5f98700832c1c79f464b3d88065b9a57702805eb0c80cbaab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 2.137903729919344, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:32.278027+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f525b738-de65-4664-9268-b576635647c6", "result_sha256": "560c97a8528d1f452c1a17832900c883cb4942fbeedd87ad83060c1639ce51c5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 2.0812728740274906, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:58.777332+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "069d2b15-4c16-41a0-9811-eb136ea3d820", "result_sha256": "86bef9b72e58c165961ede3109bbdb7ca3ac5d17e6ba4c23e256313c54693e2d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 5.496777446940541, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:43.829177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "10f2354b-c6ad-4342-86a9-f88b76e37054", "result_sha256": "ad0c146b28e71a4925ac0b4b5b62ca73819e6427234efc5b04bdd5af13552029", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01421, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 2.183314824011177, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:14.358729+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f9b0833-1b6d-4739-81d5-0ffceada2803", "result_sha256": "0fffeeb539733d6f943b20b37b47b98ec662f8bd566c42ad26bff310fbcf797e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01572, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.5095107610104606, "model_name": "claude-opus-5", "output_tokens": 42, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T12:08:54.807656+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c53b7dd4-3987-4c22-ab3a-90678269a280", "result_sha256": "4d49db4fb62b3f9fe938c1addec6dca7ed9badc7c440976f1ffd65173b03cf18", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.019945, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.137193478061818, "model_name": "claude-opus-5", "output_tokens": 211, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T12:09:39.481222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0841a79d-cf63-47cc-aa7e-b3ce21b94a7f", "result_sha256": "b538f0119cbd99425a80fa1126e056dca7fd0486026844c3540dced69786153a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016545, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.0830294879851863, "model_name": "claude-opus-5", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:55.688263+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a81f7578-7d11-483e-8b8b-b8329c6f644e", "result_sha256": "8502c4fd00391c040b563aae866c5bea72c7a2b1e9220391deafbc93f7531f20", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.026495, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 7.470003409078345, "model_name": "claude-opus-5", "output_tokens": 473, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:00.237181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "85f964a8-f9e6-4315-9a0b-dee6cbe0f122", "result_sha256": "d210898dc923ff479880e2ad2c46f2d132899bb0d46dab9f6a49acdad3bd59d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016395, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 6.198671592050232, "model_name": "claude-opus-5", "output_tokens": 69, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:17.581879+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f501d62e-473e-4837-b75f-2d33c2ade6e5", "result_sha256": "99eea43b42ad9780702a8e4788eca8a42b96fdf79ea007c2217f34d9b4242659", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02057, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.25184005103074, "model_name": "claude-opus-5", "output_tokens": 236, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:32.626434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0e64be8-b866-445f-806c-b5bad2180ece", "result_sha256": "181f9cd9550a32d38936aa891b1508c959a6c84c70f1ae3b6a420ca7b186f895", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016795, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.698781384038739, "model_name": "claude-opus-5", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:24.352638+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "13d79aae-1529-4b2b-b47c-7e37b36f9979", "result_sha256": "8df39131738f62a748b6377ee64ec0af119f807922497ef39e0b02d551d6b54c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01757, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.2535108840093017, "model_name": "claude-opus-5", "output_tokens": 116, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:16.531031+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2addb9b9-4d91-4bd2-8b8b-e8fe38c65ba5", "result_sha256": "32fc43e356f7664e6cca577d951842eaa99673a668d566f5afadebcf746b7f23", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01667, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.8696905840188265, "model_name": "claude-opus-5", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:21.861210+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d564c417-717f-458b-bc7a-c7d7f42ece48", "result_sha256": "189a7de3737df16e883263554b6f23f7dd6adfb478e280462b93a2ba021672fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.022745, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.977127359015867, "model_name": "claude-opus-5", "output_tokens": 323, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:23.693348+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "149f6c54-d011-49a0-88bd-bb6b410dcdd7", "result_sha256": "e2470573a9515f188ff9fa90e18e61933242ada5b22d6448b7e29068625abaf4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016095, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.617702503106557, "model_name": "claude-opus-5", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:49.227391+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8b6a532-8e66-4f75-af20-381e43bbcca8", "result_sha256": "9511b4719357cc43c9b47ad8b824b5eba957c8fd86ac6ae38b7f67896a7d595a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02292, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 6.009647022001445, "model_name": "claude-opus-5", "output_tokens": 330, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T12:09:23.879657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2eaaf8b-7331-46ce-bff8-e77d78bbad16", "result_sha256": "da3a57ba0e9419e03741b3aa23786c0a37369494337e0c4905d9c14ca3acce63", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016795, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.8407287509180605, "model_name": "claude-opus-5", "output_tokens": 85, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T12:09:45.184836+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "546a56b4-618f-457d-8642-40181d4e62d5", "result_sha256": "7fc661bad8b3fa0e4d8d1acbb33b7947a173f6d74376b656355fbb5a2cb627d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02292, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.889642588910647, "model_name": "claude-opus-5", "output_tokens": 330, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:46.091696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d5f0650e-7a0e-4393-851b-4b860c4dde3f", "result_sha256": "d48b23c3aff36f2206a095d48dc04cdd64b53b854a308e91acb9e9fbf5f45660", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01672, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.107550722081214, "model_name": "claude-opus-5", "output_tokens": 82, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:38.503798+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "40d282fe-346e-42c9-9ca7-bee67ca70c9d", "result_sha256": "773496427700bc99e3d248548b4cd40389b51d605c78fbbe56063dbc3c33c010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02002, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.244075958034955, "model_name": "claude-opus-5", "output_tokens": 214, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:35.334268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7138c0f2-cfd1-47e3-844d-fadc0f5fb037", "result_sha256": "fb0d274dba2150357cde5f4bd0370b27160eed6c5262f90e88f024876d099755", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018045, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.623408325947821, "model_name": "claude-opus-5", "output_tokens": 135, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:40.397961+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0b240a8e-01c4-4970-bc07-c8a0658b3613", "result_sha256": "b28cb024056aa05f5d5453517390c49d096971a7fe971ccbdfd5c1ae27daa52a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02027, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 4.4942588590784, "model_name": "claude-opus-5", "output_tokens": 224, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:25.237602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c32b50d-e41b-45df-b620-223b2f993bb2", "result_sha256": "707057e167145b4ddbdc42198319f1c55b73f4b26a67d4c23cb03599b5818056", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01817, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.449233538005501, "model_name": "claude-opus-5", "output_tokens": 140, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:02.639165+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9d8fcc77-8cad-4bba-9074-c906dbc24bdb", "result_sha256": "bae0a32540b0e129bc566891f46b180c3b558f3975a07fb1713c246a4bfc11b8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02277, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 6.483687781030312, "model_name": "claude-opus-5", "output_tokens": 324, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:05.181332+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "632eba83-efbf-4ad5-9840-2dc2fb78cbdd", "result_sha256": "ba43cc6aae48d69628c6e347e1df0c9a6231cd6cbdc4e89ca0b1b59ac3d63791", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01742, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.327764311921783, "model_name": "claude-opus-5", "output_tokens": 110, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:59.823338+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b0a30887-f8c3-40ee-b953-2a85874d8614", "result_sha256": "32bc3ef562bed3e8116cba474900e246330eae573b6ae9abdb7f277cb4e38c02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017845, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.7716471750754863, "model_name": "claude-opus-5", "output_tokens": 127, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:07.921100+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9e136342-4843-480a-9d9f-b90f4565b41c", "result_sha256": "5f944d1bc019c3fdfcb1f1f15e3da128ccf76ba741b0e760379c6af62954a750", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01972, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 4.604339350946248, "model_name": "claude-opus-5", "output_tokens": 202, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:34.720538+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7dc4906f-62c6-41db-886c-f9246638c5a4", "result_sha256": "a3ffc99753047e7e7e864d48002a371094bdc39df56227ea12cc5bb0704ce48f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.020095, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 5.548398558981717, "model_name": "claude-opus-5", "output_tokens": 217, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:55.964677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b648551-72b6-46a9-bcb3-5d6f8abe26cb", "result_sha256": "9ac0df609e932d8cbd5a0d7b553a90d71d5e7f5e536df0a2f74e25997b85f19b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01432, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 1.9085243979934603, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:07.213409+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c7fc2e5-1b4c-4e85-b0cc-f4a77f5f9740", "result_sha256": "4e51cec766a00455c3b33423581eb7c9ad695ff4700c02dc13fe0fb1ac758270", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01587, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.703928622067906, "model_name": "claude-opus-5", "output_tokens": 73, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:58.595890+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "73fea990-1656-4dc2-a237-55b16be176bb", "result_sha256": "6271e877b7530fbbc4a2e920b0194b4f8a9b4d66865b562992b1593ab3bd33d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015445, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.8830589200370014, "model_name": "claude-opus-5", "output_tokens": 56, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:10.866695+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7cacec71-b753-4b19-af67-3d3c92176c2f", "result_sha256": "57b7742649423b1e117cf6ce740469c47a9c8bf53956ec6595bc2ca2497213ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01432, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 1.866024354007095, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:50.784023+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b434f51f-f68e-4ff1-8f37-06774423f657", "result_sha256": "8a12ceca5019b9cf322e9890a0511219e13ef72d8a0396bc4e0d7ed76fb91d97", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01547, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 3.03207984007895, "model_name": "claude-opus-5", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:58.749529+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c8268d29-55b9-4121-9b28-2ff300db29de", "result_sha256": "c23b198a2064aceccfe16256984a1d119f445ab0b785209d6c388d8c2dc8b79b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015895, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 3.1681147660128772, "model_name": "claude-opus-5", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:59.682635+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "98628767-7173-40c4-8359-171246a196b9", "result_sha256": "594a75fc7294ace3e7895defb16a1cd9177db15ad618bb0d9892b94bf29e9fe7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01537, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.795630228938535, "model_name": "claude-opus-5", "output_tokens": 53, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:17.947336+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9001ba8e-8181-4460-95a3-bd6a1cffe027", "result_sha256": "9573a5da89cb9bc495d75f99a3f650ac47af5ae28d74e41029a1de66c4d4637a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01432, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 1.8323320890776813, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:34.921104+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c5dd92d6-679e-473d-98c7-50b86cca7c82", "result_sha256": "b8b590d9c1135036c800ed9fdcd1d568ed05f3ae5f64772f68e1bf95b31fc644", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01537, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.8876289980253205, "model_name": "claude-opus-5", "output_tokens": 53, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:06.597392+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a364e961-bc40-4b61-a6cb-ba830817d491", "result_sha256": "763d8569d4ca186682d202d1e0b9db12876cfdf88ac868647d4d1abce048b4b3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016045, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.6665223359595984, "model_name": "claude-opus-5", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:32.257847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "fba53810-7a75-4f37-88d0-5fadccc96184", "result_sha256": "78288133e78fd135602b2d301bb0cdc75dafad711029b9fe4e876f4834e45745", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018745, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 3.719460439053364, "model_name": "claude-opus-5", "output_tokens": 188, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:18.343867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b0746bdd-2abf-448d-9f95-9c48684e74b2", "result_sha256": "8b08fd9e4967869ad591dc544412a0671046583caf0c1d3163b42c1871ddf5db", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01967, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 6.0006373500218615, "model_name": "claude-opus-5", "output_tokens": 225, "prediction": { "choice": "A" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:52.774199+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "097d20b6-5f36-42c1-9918-8ab307876abb", "result_sha256": "baa7151eb8e1b73717feaaedf736684adc2644048027278a0e40e1f7ac4a6540", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.2099071469856426, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:08.514898+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d555633e-41d2-48bc-97c2-d0fa0f6186f5", "result_sha256": "41a6ceb4db2f5b708034ed2b8aca83094ee73cded0798ea4c19e86e5f93955b8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.435085572069511, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:23.238803+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5f0e3833-8609-465d-b2da-e3d22d715f5c", "result_sha256": "089965d5accf4728dd9d007ab4adcbca8ac3cc7da8922753d6aeab72dc4d76f5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 3.2748661399818957, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:40.560434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "69dbdefc-9a17-4458-9f84-8df4ac9ab419", "result_sha256": "0467fec97ebd382b7401d82347d3cdbdb66742f58d3b2ab157afe0007b7314b8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.45995413698256, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:40.964187+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a8634089-e337-4a76-95ba-ed821d53b2eb", "result_sha256": "0ea049a1cd0cbd7e7b5107d710f14269225ae2fed4c1c68e26febe1eb34b6d80", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.019916226970963, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:45.144149+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2f526e2-b811-43da-b594-9c9056e09f83", "result_sha256": "77cc8b35c79b4008e22a5bc1e63a2ba045ecb42b1647aa31e5652a93d8e8c8ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.3149605629732832, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:41.565399+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "aac3f068-2d4a-48c9-946d-8d2ee01ed3ef", "result_sha256": "befd24435ade80bda868b12cc0f3d92c6518905d869b864e65cbe0e6c6817ef5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.1587923150509596, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:54.341737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c0ff8cd-fad9-42e8-ad75-f91780178f6e", "result_sha256": "6daf390bd629c34c0ddc58c81bbb985818d971f4c8537335e87c1bb04cc2e6cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8538779829395935, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:58.008955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "cde603af-b870-4ba2-aa9f-1b613c4040a5", "result_sha256": "30a1f85fbaa364be11033ccfca78a56767faaa10668327be754fa06f3be07cd7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.394972661975771, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:36.616073+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf37222f-ae65-4442-9407-2f9d62fc54e0", "result_sha256": "8346caa976116bf7da185b6305c2551b00a305df65a5700c1fe7e9c083e77baf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8169512719614431, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:47.070156+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "31b2930c-ebbb-477f-adbc-8813fa405877", "result_sha256": "aab7b921880db05d8f29ae6f985e3bb50e6ee38d228468cad72143c692c1da71", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.6399153870297596, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:32.359716+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a4eef379-9956-44db-92ad-3be42f23d59f", "result_sha256": "7f5b80d30e407104d8537fe7cde3ebfe0241b3d4881eb2016997366c3937dd5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.2470681429840624, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:08.128372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "04c2e938-a9c6-4e97-95b1-863511c13072", "result_sha256": "5fb72da72a7acc06613625643349df9772d14174ef78b8b7e56b55d902d21d6c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01618, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.698219929006882, "model_name": "claude-opus-5", "output_tokens": 82, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:03.095170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "44fd1fd6-acde-4fc3-9791-1edcaec32e52", "result_sha256": "b230061b6a602d03e80663173acaed89704780610379c9c3a5f2cdab650a6955", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016105, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.0845181989716366, "model_name": "claude-opus-5", "output_tokens": 79, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:11.930177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5f3e26e0-3bc9-4248-aa60-6fa9451af1c5", "result_sha256": "d80ebcd746eeb68f65926ebdfcbffc44d0950b94be251352aeba974c8b5a596f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01608, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.622772760922089, "model_name": "claude-opus-5", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:11.757599+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "db30dfd9-0dab-42a1-b7d6-87d86de6dec8", "result_sha256": "c9df2d1ff05376496981b48e146904317e582e7e5caaa3aeff833317666e95ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016005, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.8726579380454496, "model_name": "claude-opus-5", "output_tokens": 75, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:27.511814+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1d4b12ca-9442-4a92-8122-0d2df25c9a23", "result_sha256": "5b30966c396fa3c3fbadfd8eeb7e41f2d68115a89d243d40caeb4690b93a23d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016055, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.64645134692546, "model_name": "claude-opus-5", "output_tokens": 77, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:26.304028+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9697c080-d81c-422b-b2e6-b4f0070706bc", "result_sha256": "caf6bf11af56bafeb90e3d24caf7a1799c737d9f805ec37eafd30e3c73c590af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01598, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.0001188169699162, "model_name": "claude-opus-5", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:38.465008+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea98de3b-2f74-4d99-b941-a18cc27bed4a", "result_sha256": "7cf03cd6114bf88b4bbc9cba2bef49d7da4d28db46235b1a67a0a37e5b28b527", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016055, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.4887738339602947, "model_name": "claude-opus-5", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:57.263908+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "faa206ff-7d87-4f53-a142-ec90a416e313", "result_sha256": "8c4fbefc2970905491cc7d141ddffaea883dc161a2b570b64d4fc3e4fa0394f7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016255, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.5452271009562537, "model_name": "claude-opus-5", "output_tokens": 85, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:25.739860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b11edaca-4777-4057-9a54-f6af3019c845", "result_sha256": "f3f9c543abfce5cf33d01860fa67dc3f64e30ff404d145579beec263f66b31d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017455, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.8455820289673284, "model_name": "claude-opus-5", "output_tokens": 133, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:36.578133+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9161048-90e6-409c-895e-82bb7c707131", "result_sha256": "b3c4baa1c8b2db810113f347fe9a1c41a32c902c1b6d3efafafc797096395ef0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017255, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.2914304269943386, "model_name": "claude-opus-5", "output_tokens": 125, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:15.504311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "574b0b95-f9c1-4676-b407-9c9b3bb1c42e", "result_sha256": "ba3fde8a9ec29fd878b953d6858458d7d8d55c023b9ab12675f708c13319572f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.66246646491345, "model_name": "claude-opus-5", "output_tokens": 102, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:07.522619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c6e97f0f-557f-45cb-a29b-92ddf5dedd4c", "result_sha256": "63664a37178c1863473703fdf485428750507f454a85017b255a0ca60fefe8ed", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.020355, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 4.701592938043177, "model_name": "claude-opus-5", "output_tokens": 249, "prediction": { "choice": "A" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:21.230148+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "24599603-5e27-4e78-bdd9-d90a0e7a0a49", "result_sha256": "d98bdd52415a8cb15b22aafea73389bb50135032eda7f9955a30362a3bd8cfbe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 3.4238351460080594, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:04.104328+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "95de8ada-716e-48fa-85d7-b15f2c319ea9", "result_sha256": "e7bb5a8ca3a06fead1ad8dc7c4a03318e727fb528423228a04c89d578ada8d22", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.5571884739911184, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:10.627365+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1bc502e8-b004-487d-8671-37b7c9f76f25", "result_sha256": "6aa4348bd95298d73edef630affaa2c0d6f913751a64dd47d018a278a20ad278", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.211859790957533, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:01.083232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "eded5bd3-c37a-432b-8fcb-b34dc5efb65e", "result_sha256": "222ce1c311a73ddda4c22e49a25886f2bd7fc70890ea353631cb500c4dccceff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9246367890154943, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:51.505559+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1de18dfb-b79a-4648-a36d-789684e9795a", "result_sha256": "96599aebf82f3e6bd497273cfda16ed29347b3a75576ea4202356393508518b2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.189311734982766, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:30.079529+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "84d9d7ac-7f2b-41e3-8a63-198600ef9ccf", "result_sha256": "fbba78ce11998824fe7b7ed5ec3be0c38a2bc0c31bb42f52cd073aed3f3037d4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.6924663690151647, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:44.159515+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0ad0c73f-aa4d-483d-b18f-90bd517a580f", "result_sha256": "3d5a30ef9ee91c5ba5585dc49711c19d41d73538890f899961fd99462f0205ea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9859111880650744, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:49.611712+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c5926544-f74c-41c9-a5e1-a8eacef96cab", "result_sha256": "3e2b39ad2439ba92f04972276731f6d55b70ea823cb9d4f334b61fed6744b268", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.278180051012896, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:03.211800+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "035a1816-c4da-4bde-b299-0446637b71d5", "result_sha256": "924fd717a9d9cbfdf08ed00883f0c99b88ad3cdc7c97635abb6054bc7994d01d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.3728753379546106, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:28.380252+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d4581dc-cc62-4b57-a66e-ddbb196e4737", "result_sha256": "cdcc68ca11eb276edee58684c5661b581e222598247e13077508a0e31a0541d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.027457650983706, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:25.065717+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8abf90f5-a2e3-4ee6-8381-b3b85949f5e2", "result_sha256": "1d3e3363e497a06cdd45888d3fd8ad227551d41fab4ff4cba902d4e65b3c1ee1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.0427591720363125, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:51.701683+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2e730ad-c3f6-472d-9418-6c377c0b462e", "result_sha256": "97bb79eb006d6f64c39df83970ddcfe3e068e39ad497e1b180357310ec8e7e50", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014205, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.820346469990909, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:50.519034+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4e2c0a4d-7d34-4d18-9907-d89f381a53e8", "result_sha256": "4d7904bdcf4df32be49fc61375f78aa58096dfc55326c90b7d0d10fbfb89c0fa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.0556223950115964, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:53.857000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "abe76c1d-7e40-42ee-a5b0-2775024d767b", "result_sha256": "1ce78105a814cf9b8369cc6d379d2f9ff8088a5b856dca9e382243c45c2cb50e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01544, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.7679634500527754, "model_name": "claude-opus-5", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:09.208514+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "04f622db-604f-47c1-a7b6-67eaccdda9b7", "result_sha256": "65eeab5acee45609d254dfad5b9f1ae76c355c94608064ba6f24d128ed20ebe3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 3.197108093998395, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:44.145387+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "42a94df5-87ae-4563-abc6-af84e8a4b3e7", "result_sha256": "d15e7e95120550e7b86966915657c9f8a03b0cbf861cda1f3accb7da9acd4b54", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.223421359900385, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:51.287639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "01295b23-3a6e-4c98-b5af-362300fcc9e1", "result_sha256": "a82bfe5a49aa4a1f3309ed5a037ae66f20b944b5677dcf544a3c3d4c0e10ba36", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015815, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.9021712449612096, "model_name": "claude-opus-5", "output_tokens": 70, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:25.563817+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d84ae96d-2652-4ee2-9ee6-7da15c289a55", "result_sha256": "0e7bf81c39f7f4ea7a223af06c299fd0946a682d3ef92b93154ff9400382e67e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.1388372209621593, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:16.982909+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0e4ac23-3440-4cd9-903f-9a9e31ca6549", "result_sha256": "2214dafebf191cc5527fce55b1c6517116aa85c3ead055e9780872775f8f1cc1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015615, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 3.0100152710219845, "model_name": "claude-opus-5", "output_tokens": 62, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:51.714143+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "33974c74-252d-4708-9ce9-5acd4dc71fdd", "result_sha256": "d812f11ad468318c61dd5f4b8a61fd651c22e06db4d7dc62eb0c549be19dde08", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015415, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 3.2606541459681466, "model_name": "claude-opus-5", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:40.413269+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c985006f-54eb-4462-96e3-16959ff83271", "result_sha256": "768a1359fec781102e80f95db5c02fff4af71b9283ed216ba92342e690b4956d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.9480436759768054, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:54.952195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "230d9e5e-5472-415f-81d7-da5f247e5717", "result_sha256": "4b0f74bf4ea558cd70b99121a900329e88ae94ce50cea2755dcd87e4133f04ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.5076094190590084, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:03.931681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d97299ab-b2a6-443d-aac8-b3fcf86773d1", "result_sha256": "5ab72fedbb1359c01ecf1770e0528a844ca534762873f309449da3892de356e6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.1315712248906493, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:25.540589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a38eda3e-9a74-4032-b965-aaff5a6893ef", "result_sha256": "2a92f54c92dccce8d0db2869aebce8101d4b5f234065d92c54cd9f3092733629", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01434, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.9867992829531431, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:14.236147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "50555940-eeaa-432f-b605-c7e25585c980", "result_sha256": "356a5f1d925be873c63d5295d3317f9cc83c1a37ec91ff51be6fa6ad228e7b1b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018065, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 3.5916036850539967, "model_name": "claude-opus-5", "output_tokens": 161, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:42.889470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2208e5aa-3021-4fb0-88b0-e614efd5b676", "result_sha256": "1626c9a65c703ae45428610de3d63db46784b975b4f4bc65c9338b68cdb701f3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01584, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.8614589099306613, "model_name": "claude-opus-5", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:14.936378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6f18b9c3-f7b0-463b-b85b-244b6b988cbc", "result_sha256": "b579363646b212ed8a94fc72d978c85b68909860a0d59e45640b9583e46c10c8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01779, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 3.206112814019434, "model_name": "claude-opus-5", "output_tokens": 150, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:28.740439+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6241930a-25d5-466d-b98d-6eef3bada933", "result_sha256": "98cb670dcaf331f5750f6ee8bde1e614c8d5afa54d452ec48d325bea0d8af875", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017515, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 4.401619579992257, "model_name": "claude-opus-5", "output_tokens": 139, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:10.341857+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f837bd9f-672d-423b-adcc-f24114472a8a", "result_sha256": "7d4132e62d9c9ace11ab74fa68cb632b74f859a71a501adcb9a80cf17caea930", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.9075438419822603, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:06.079756+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "87b655c9-bb7a-4ba6-9ae8-1a4bfc647fd9", "result_sha256": "45492ced5c46c3ea1b2197b71e794bb83704dad444c98107398b1597fc91eab7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.9336639059474692, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:43.806869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "325b7e71-dc35-4329-8bfa-583a0318f80e", "result_sha256": "158a6f16ddbb2a259e1a0342876ce9996db5882147f977ef649129aeb29f7a94", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.873793622944504, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:16.010350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8a642eec-e4bb-4074-8868-f607f211609c", "result_sha256": "6548e54adceb2405e2cfb62a501ceb8f92c431c8d91c74fc90b03c0ee5332e9c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.845548804034479, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:29.141869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b470d442-b0e3-48aa-994e-65b257a1d153", "result_sha256": "a34907da656b0ec29f07b0b7cc68233a6d6c1e08a7257b11e13a576de2945bdc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.519625935005024, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:43.776341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c110db81-1172-4ea9-9e90-baa9a97ce84e", "result_sha256": "587b984ec675a3103f838ecd9cee69f87693c5732b07940f60e7a052c2144e8d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.9101696560392156, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:20.764138+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0cefabc-48b3-46b6-aed8-24e26c564e8e", "result_sha256": "3f71a17e6af89f942d444d8416ea1c434cec986da8b645d1149605783d6ed0b0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.068556701997295, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:45.060549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "63bf5cf9-7d42-4501-a019-a4624552174f", "result_sha256": "7f9251db7c6f03499466fba5e31a260bd6fea1ecf62fdb2103e1ee7db3ccdc37", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014315, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.8663034898927435, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:41.108365+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3911542e-f51a-42d4-b0e4-855302164f3d", "result_sha256": "0bff24a14d90964ed67e98ecbd3b9c8ab278b502ad6cba3f7ade7d619aad808c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015745, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.739813438965939, "model_name": "claude-opus-5", "output_tokens": 61, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:10:00.560944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "dad59a82-8f1d-489e-b495-55fdcf24b03e", "result_sha256": "4f07b0d31405e318cf2154733554b51c60843f6d25278b0d56af8620f7fb1cdc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016345, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.5524460420710966, "model_name": "claude-opus-5", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:25.838261+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a5b91e3-2513-45e3-8aaa-e38517678206", "result_sha256": "d56d4bd879e75dda2b74f828a4dd8d1d89a430d5f1f436ab2a32c8eedd10fc7d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015795, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.651968408958055, "model_name": "claude-opus-5", "output_tokens": 63, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:21.476821+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0704986f-02cd-408e-9a00-2c02e62ac3a1", "result_sha256": "6ca17477633b4fb4b138227fc94bcc9c91826a48e035735f9c35d59d40a98c32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016295, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.975410785060376, "model_name": "claude-opus-5", "output_tokens": 83, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:37.920482+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2794c891-4edf-41ea-9543-059e897f74d9", "result_sha256": "af564370decc0627ebb9d1de1235595d9bc6a5490c4311dcea7554a97026ee54", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017495, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.7770229120505974, "model_name": "claude-opus-5", "output_tokens": 131, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:46.917743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e987ea52-07e9-4a39-ac46-54c33db9706a", "result_sha256": "1ea4db6ffe0c887145dfb4e2a0bf78726ef62925c8351df8c5d3ed62ad918c94", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01852, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.507633361965418, "model_name": "claude-opus-5", "output_tokens": 172, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:09.806392+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "06b312d7-d5da-4224-8901-bbdf65a453d3", "result_sha256": "740d07344d13a9823c2aea6d0046d495d59399d379ad46ef5f25e5eb3997efa3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017595, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 5.011828515911475, "model_name": "claude-opus-5", "output_tokens": 135, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:45.410077+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "73363b8e-4103-43ac-b7ec-6478588fbf07", "result_sha256": "35e3a421f8f9a7daf7665bc62bca560c6f23a8210b6148f6851bb331b1c13212", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017845, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.519892710959539, "model_name": "claude-opus-5", "output_tokens": 145, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:53.506520+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0baa5a25-5abd-4c80-a360-31eff9517028", "result_sha256": "6420d1faf879e158625ce1831068b72e32e07d92c2730eb16c277c61c748ca59", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01562, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.7517399759963155, "model_name": "claude-opus-5", "output_tokens": 56, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:38.331114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8ce5aadd-2d8e-4a03-8ae4-42c93040f464", "result_sha256": "fc28174a85634afc3ee832838d21e9956f715038d297ce9f031b5781014c5af7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01552, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.9586243729572743, "model_name": "claude-opus-5", "output_tokens": 52, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:40.805845+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "02f4ba2b-7fad-4a14-87c8-44b0650c45be", "result_sha256": "22f0ac82939d9546b685c44ef3306455cb5130b4a0ae48e6503f1f402dfeb324", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01557, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.383471436914988, "model_name": "claude-opus-5", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:15.991775+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a2f9cbd8-dd03-44c0-baa2-3cce09650e63", "result_sha256": "b87d07ae45ac7449c6cdc3997049169fe085d8d70cc00007b7431948ce9d26b0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01557, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.70322943397332, "model_name": "claude-opus-5", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:32.967586+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "25633b96-0f3f-40f8-9ddf-41034093e81b", "result_sha256": "b163ddd136b7b5cbd75b92d99ebe0456b29f93b1190f32d218d68e03472179c0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.4945498630404472, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:48.535772+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f08a9f4d-74f7-4b68-bc27-da8050d12106", "result_sha256": "7295fed34ade917b878cad63dec2bd2bb65d590c5585a129a0bc82435c58abdb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.244341160985641, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:00.379588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "da19da58-56dd-4190-8a96-c52cbbbe543b", "result_sha256": "a20d03226956dcac92b5853bfe63e3913b01e1096d4bd77b4a1d5249d79df2c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.9839594309451059, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:21.499364+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "472eb7a6-558a-44d8-a08b-4716d17d6661", "result_sha256": "0fbf13b3c399d4f9975b14c1f2e1114ff108e9b4e56a9607477ce066e57a5cf9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.363664577016607, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:53.548273+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d784d54-0840-41b3-b632-53a67500c25d", "result_sha256": "1b94d6f6465a2d70fd65efba5954f2fb71e5fa9ab92587bc70084022b0628cc5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.0585083820624277, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:45.956973+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9cc45ed3-1d58-40da-9b6b-941f03a5cba6", "result_sha256": "dadc03855aa08a6a0afd05cdf3682e66d58cdd2ba9089b5e2d02ba7354765163", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.182809791062027, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:19.653591+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a2af3cea-8ad5-4cc0-885f-e17ab29c6de1", "result_sha256": "42e6dbe03ea6156971183a3dcf369ce69ae4df265052cd38c4d105d9bd4f90cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.3093039579689503, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:26.941830+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "90ab0a6d-5818-4209-a96b-2689f00a7196", "result_sha256": "88feca4ac0cf553fa3a1d97d6c07b2f38b160162228b904c5d1582d7f2616cd3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.9262015679851174, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:44.638797+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6fef4f94-6d29-4dfe-ad1a-d8e790786cfd", "result_sha256": "0ec917aa7f383e7799ed3b34dfdc31822c803da023d0f2848d2cd333942e3868", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.114649279974401, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:08.376137+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "91ab10e2-96c3-4e30-9a1f-7f89455f98f6", "result_sha256": "df83292b2332019fff52ddd9b682683393353e7ed6b460eaeeca3f8b2010061c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.9540644839871675, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:10.642934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c0b0d078-ffb4-48dc-a461-22b86e8fb6bc", "result_sha256": "0a66add21dcfad00459e5df7dae753e352967e70cd88512ab5f5ec1184ee2656", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.5903819140512496, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:33.393104+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c5a7d27-5b06-4afc-8d3d-e4a28b8ff083", "result_sha256": "dfd5d7872146912ca35472b4f003fc68fbe3c33e383560edc7790661838feb3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.333905070903711, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:55.469112+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b2734b3f-e665-4775-aa52-c5ffc139950f", "result_sha256": "e08116698c4aac0b00632b9554004b46c0f1c3d0e5df31b5015b2a71303f0b48", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.440799474949017, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:17.175833+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5a61a40-73a1-4a65-8f57-52ce52f89004", "result_sha256": "ad38497cbfa97cd4d22469db1c38d50d3ddc8da8148d9b946572ba6832ec16ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.2101724159438163, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:31.715917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7d4c33c7-2c75-4db3-8176-8a95de8f9d35", "result_sha256": "f646e74c0e96bd79c6b3416cbd22f08cb7953f6ab4cfc7004f30d586d80eb120", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.3954857600620016, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:39.254338+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5801a5fb-5397-409e-b2b3-5487c90b162d", "result_sha256": "f7f5573563ba75bc04823c1d20c9af142fc28b990a9290d82020a0395d478713", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 8.837906496948563, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:38.840364+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d7577e50-5ed7-4617-9c41-d3dfb541d002", "result_sha256": "25f59b826937b34e2cfb7c9fbddca94aeb4dd996629ed9dfaa1cf6b0a2eca33d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.172266007051803, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:12.493664+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9a796da7-be0f-4811-9ab9-80328c22e93c", "result_sha256": "5124c27754bb4bb0a29537615f5cb48e412fa82ffbe41a700e86b63d001e8e67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.3726321069989353, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:28.824420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d84dd688-d9e7-4d85-8d55-f39523736513", "result_sha256": "2ab8d6b0892e44584589789022f598c083916ec95f033c1ebdb1ec6b87fc517b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.059077264042571, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:53.284925+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b1bd90b-67fc-4a42-aac7-1fd74ea58995", "result_sha256": "cefd042463ab2389b6e9b7f6d9b886060bce6ae52b1642f8120670f49696e822", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.093614288023673, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:46.238127+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "616cbe8d-86ee-4bf5-9716-fd278ab508d9", "result_sha256": "66ccda20881792a0f4954892c106aaf7e699b8aa0da834e2027b74b12e7af8f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.268117563915439, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:27.966044+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff914b08-6d51-429f-8fd0-2104e92e332d", "result_sha256": "d8b570c0cad17543d3913985a4080d1734d68f2f18ff1c4219ff9810e9f9edeb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.052003897028044, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:04.416981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "044e26c8-4208-49e2-a315-5fc7fa63cad5", "result_sha256": "4802ddd1f8fde6f6e1490a964e4107748870288ce824471ed70fba5e2685ec42", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.294426295091398, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:37.635882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "fb635e5d-74d3-4a5f-96a3-bb1bdbded703", "result_sha256": "4db2a4103d392815716e196bd12690b62682e2b6eb4ca9cbdfc2e4e0c896b6c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014405, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.123782537993975, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:06.032589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d241ca25-13a5-4480-a41e-70831d04b803", "result_sha256": "3bf6f2fe9c153328f1ffdf1f07597cc73efa90c8b1f2de8be42467771622f7bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.732013701926917, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:22.814274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "612a5bd8-976f-4ae2-ba7e-31439e24ad13", "result_sha256": "7a960929163f5130f5f22ea00928232766d0ea0de41d5fbc55bd406033ed09ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.2720221279887483, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:35.096912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "35319c9c-f290-484d-be4e-bb6cd6b69885", "result_sha256": "f10d4e5196dc20af7df533b92912190725a837f75af287c53c5345d75943fba8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.2765339529141784, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:27.777031+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "aca3045d-434a-4190-b86e-6b2a955f0646", "result_sha256": "77cabdf7f33e675ae437ef40c4dda0a34ccdfcacbcbb4f44a506615ab6272811", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.405922662001103, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:20.217239+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4da01257-5ed0-4521-ab65-00137f174ecf", "result_sha256": "e9bf2c1051abf88e1c186bf5bdd218408d07dc2897b9ea0191e206f8ead69f9d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.9114576099673286, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:04.830759+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "12bf4531-5ff1-481a-a685-52cb196f1020", "result_sha256": "2dce0c4de8bc60f72819df12044f6d4d28edabaad63ba022cf2098803d511985", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.5220339350635186, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:03.245497+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b2411a3e-39ac-44a9-8ea7-a2147e43d673", "result_sha256": "df446e3705f90c5ca76a5de84ba82ebbae6f5bc7f6b008a67eef786ac7d9c476", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.0206846160581335, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:34.129877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a71d28ab-2b2d-440e-85c9-5a56e75189a0", "result_sha256": "d4dd690c9c6281c1e79f2bf1f01c952b6885f8b38eb137666325254a1e2c378e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.8543293359689415, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:23.909892+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "11eaf48d-613f-4659-91fb-c8fd03239a62", "result_sha256": "8faf3ffb72db6971e6090a22a24b5a2a7ef0e2a6539924c95c06fd13a9f77a94", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01447, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.1188946830807254, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:01.556022+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5eb38c68-7e6d-4bcc-95bb-18a8dd248277", "result_sha256": "8f6f17f94b731f8e1fb19fea7c46664564f61ebef60b33caf4574c97791bb34f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01662, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.8632953109918162, "model_name": "claude-opus-5", "output_tokens": 97, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:28.758379+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3abf33f7-78d3-4084-ba6e-23e5ce904d0d", "result_sha256": "a43f634c3a73f1bc6ba6ee83cedfddc6ae67e7844528a3034a66a977d335d66a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017195, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.3562364609679207, "model_name": "claude-opus-5", "output_tokens": 120, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:14.664074+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e43a1b69-da14-4a3d-a5df-634237952c8d", "result_sha256": "1c43d7bac99dc05f17e464f7f63f00f8d90d20b8612a6cc5930b547ce9381e94", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01602, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.767312241019681, "model_name": "claude-opus-5", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:42.341942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d5846d4-51d4-4202-b85e-d62e7dec0b05", "result_sha256": "c3c099b8fb482734623e122491873abd337e6060b41295eb8970146d4922f230", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01572, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.703425161074847, "model_name": "claude-opus-5", "output_tokens": 61, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:07.141222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8000a969-1257-4cfb-ac85-881195eb91a6", "result_sha256": "b26b004ac271c618db11d2260efd15ce3c6c84d2a4c59f4c32a3fa823194beac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01677, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.294167770072818, "model_name": "claude-opus-5", "output_tokens": 103, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:32.585043+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c9f0dabd-f162-4b8b-a4bb-3f9b640f6543", "result_sha256": "4855a54947851ed56fa07252e46193d30e9e0242af0e4abf119dde112d7b067f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01662, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.432309611933306, "model_name": "claude-opus-5", "output_tokens": 97, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:37.129144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4531dd9a-ec80-41b7-ae02-41d3e80282bb", "result_sha256": "288599c6ed27a620b71a1abda274fb231884919fd47c911302bb4f70451372bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01617, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 5.243524444056675, "model_name": "claude-opus-5", "output_tokens": 79, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:55.886444+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d2e2683f-c82b-423d-a2b1-1637a76f0451", "result_sha256": "7c146f06ce77722552f99d5504865fd5f2dc43cbe83285685bc8794d8068021f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01657, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.1098716150736436, "model_name": "claude-opus-5", "output_tokens": 95, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:20.185285+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6245d0f6-0df6-4489-95c7-7b21c302f50d", "result_sha256": "55f27c96c1ae47ea9b286b8ed6675a67b0bfc4b081bd6bdde8775acc1f81a04b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01642, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.6502997409552336, "model_name": "claude-opus-5", "output_tokens": 89, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:30.733723+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4e215740-2fa0-4213-a40a-ec4617e22f3c", "result_sha256": "91f772f142d83685c1169f604ccd8d6c54c6763a88a7c7e1c1468308bf888579", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01767, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.5862797929439694, "model_name": "claude-opus-5", "output_tokens": 139, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:10.463168+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "754634e7-8abc-4d14-a836-afdd7e464967", "result_sha256": "d3947f6b971a1580d317b9871488a74f7a020a0c8d8ca20aef449e27b09e79ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01652, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.937249858980067, "model_name": "claude-opus-5", "output_tokens": 93, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:11.780003+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b1629444-9dbd-42db-816c-9674ed380323", "result_sha256": "32cbaa5e71e8148ac1a1b25ccb8d0ab29babe709d667deac3ae8268415b98bf3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016145, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.901952369022183, "model_name": "claude-opus-5", "output_tokens": 78, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:45.541825+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d89f27fa-705c-41be-8ddd-4510f48796cd", "result_sha256": "9f2be5b3da4b52ef18ca74d6601cd45c74ffff316b3cf5f5867b5bd7ad9e36de", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015845, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.8246711120009422, "model_name": "claude-opus-5", "output_tokens": 66, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:58.043686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "47a73071-0c4e-4d72-832f-ff4fdfe16441", "result_sha256": "880d6ee3485ca1c50825f8ecb0a6f3d5d311c1653171d24208a1d73deeccb9a0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016045, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.9327957310015336, "model_name": "claude-opus-5", "output_tokens": 74, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:27.107597+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c771400a-25ed-4114-a1b0-513e9cb2f5b5", "result_sha256": "c3c27ad5b91c5c560b2a85e486bbc221682d04dc78633d6b43421c4680daa987", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016095, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.9417102159932256, "model_name": "claude-opus-5", "output_tokens": 76, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:47.786564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "27401c90-8030-413f-a425-bac75b4c05b6", "result_sha256": "a1524a0d6bd37f8fdca1755309536faa931a30b55e82669adeb2dbfd6249ed70", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018165, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.7310939090093598, "model_name": "claude-opus-5", "output_tokens": 153, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:57.665922+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "df9b6191-f61f-44a1-b453-d886361629fc", "result_sha256": "6ebde35b2e0fe7da32833e61c7f8934d3d58fb4b0db69da6845681fa4f7815db", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01879, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.2400063049281016, "model_name": "claude-opus-5", "output_tokens": 178, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:18.779019+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "edbacf19-7103-464d-9725-0a56e8d236d4", "result_sha256": "13b44cd9111b022e77dbd48c9edc6249db7b016536bcb0c14c51518c3a89c8bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018465, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.4543996149441227, "model_name": "claude-opus-5", "output_tokens": 165, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:11.704631+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa09f6bc-83b3-407a-9ab9-6c8618f2daa9", "result_sha256": "22f7f55aa2cb193bfafdc38471cb0d6c0ca52888d1d41490334352116e3fc0e5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01844, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 13.660711804986931, "model_name": "claude-opus-5", "output_tokens": 164, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.444609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "61a532eb-4281-40a9-bbf1-28b5b229f794", "result_sha256": "e72eb7f0f8881618e97a053b191f73df5467ed029762df57117864e64e6ce0df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017665, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 4.977174016064964, "model_name": "claude-opus-5", "output_tokens": 133, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:28.706004+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "32990a3f-30df-47e1-9645-e77129e3f0d8", "result_sha256": "0aa0b6e13e8ba602efd1bfd8d6733a21c96460d11e30e1202290ee133457e27b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01724, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.2371656299801543, "model_name": "claude-opus-5", "output_tokens": 116, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:10:04.388897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c95a7065-1556-4e35-9153-c069daaf747c", "result_sha256": "0392ee9752a786432da4c5566563f4a4fb266efd2f72615fe770c748b158ca32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01854, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 4.1164323459379375, "model_name": "claude-opus-5", "output_tokens": 168, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:15.191169+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "92deab3c-f1d7-4453-a913-04cbc6f8a168", "result_sha256": "e9b803f801741c4d3eea020b37f9d2bc505cbaf0b91e6a3da31cea7b02456dca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01759, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 4.2332485150545835, "model_name": "claude-opus-5", "output_tokens": 130, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:44.834559+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c89e424-db17-46fc-abb7-f88c1f3871a4", "result_sha256": "f8207f04adafc1fd3062940c1bd2273c7af69bf41ffaa05fbc939ba6649b2d85", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.020665, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 5.190015550004318, "model_name": "claude-opus-5", "output_tokens": 253, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:39.411727+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c4ceefb2-c3e6-4677-acb2-d03b4fe63bde", "result_sha256": "4818e8446c0c9bbe95a03acf8de99b8d65b7543f6da977e7bfd04be5995b714e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.019215, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.81181441701483, "model_name": "claude-opus-5", "output_tokens": 195, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:47.822763+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a0a3bf3-8309-4626-b2c7-c8d018c60e49", "result_sha256": "7670a2d944b692a8cd9aabad0d5e5e16f684cdd633ecf8cbedbb951b8ece1d3a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01759, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.244338770979084, "model_name": "claude-opus-5", "output_tokens": 130, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:33.093863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6936121a-ab60-42a8-8b92-8e832ed0ccae", "result_sha256": "d0033364b156f4f70f83dafaf6d8e4141c3dcd0eec288c89c28167cd6f53e126", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01739, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.323509279987775, "model_name": "claude-opus-5", "output_tokens": 122, "prediction": { "choice": "C" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:00.345513+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3150c65f-582a-4883-82f0-0e75913145f8", "result_sha256": "bc3fe7aac1e8bf24428731ae50a7f384b7241cdcbbf3977d18d0fd08585dc44d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01655, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.0718910479918122, "model_name": "claude-opus-5", "output_tokens": 92, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T12:09:20.001201+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a7dcc554-2dc3-42c0-a660-7c8e033561ca", "result_sha256": "0c93f2e4997fe9ea397b13bcd189e26a8cb6b9c55af4c7daf38282ad5e1e1ce6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.706735857995227, "model_name": "claude-opus-5", "output_tokens": 131, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T12:08:57.674981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "eefadfca-6e6d-47ad-b663-870db0771d46", "result_sha256": "f4c95112c89eee6ab06cf44d64173e8031bf168d96f4d36f0518896da896275d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 5.46353177097626, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:48.749647+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ccad6b6e-3a89-4425-9ba8-40d0fb2363d7", "result_sha256": "4b684f5573d9e0bac41dc8a20092d225443b4a74ce052d62eab8933a87b055e5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016625, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.663676081923768, "model_name": "claude-opus-5", "output_tokens": 95, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T12:09:58.592668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6e92cc65-935e-425b-9a23-3840b88a2b11", "result_sha256": "d0d4d2651a0ab777218c4b7514c78aadfa298f16d1f30f290b9087d5a673a857", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.335263382992707, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:49.068474+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "85dce5bd-7093-4ad8-935f-7ba48ded822b", "result_sha256": "25fd5e9d6841391f59e3822e7f74506761ab379803e4efd2ffee01a49af1f7ad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.577036325004883, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:38.123432+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ddada721-a616-4866-a418-b82b7c37ebcc", "result_sha256": "912fb9c62dc15275fe0d5141604cedc7d3e4ca84912601fd93aec08b343b981d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9645242699189112, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:11.466622+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "17f2a1f3-4343-410e-91ae-67aaf35b7d34", "result_sha256": "72d8fded2b121df444acaf6e50cbe37f18e0b9ff9d5a7f18c85e385b0842967f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.1835320310201496, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:14.440568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "feba0868-61ef-46cf-952c-cfd209619b1c", "result_sha256": "08c49b9c647ca0cbdd99309a7ee812a1f6ad9f2736d3b2eafa1c104acc96e675", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.0523949339985847, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:19.950966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ffefa8ea-41f3-464e-9bbb-9533194c1803", "result_sha256": "f9a9299089126e1849d0df037361d6958d5eaa16a0c7c5ceacbd0ae7df0831d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016775, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.1403631230350584, "model_name": "claude-opus-5", "output_tokens": 101, "prediction": { "choice": "D" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:20:54.469992+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d5cf1a36-6c3f-49a8-8ace-eb1ff3455127", "result_sha256": "b8ddd8e716e9d998fbf5ed89694ddaf8f1e7e7e5f9d63a7866fc631bf8d6b596", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9789902380434796, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:01.099974+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2202eec1-e4fb-4506-907b-9b5e7880fad2", "result_sha256": "897446c8ace8e6b3aa630144e2a975efa9a6b4cd5b50553a884e696bba6d048c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016625, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.900702119106427, "model_name": "claude-opus-5", "output_tokens": 95, "prediction": { "choice": "A" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:10:02.707797+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "64b38e36-f8d9-473c-9995-58bdb8b4273e", "result_sha256": "042a7022ceccf21d1d7fbc6e8f7458f3ce2c22447748b7b4ede7f1e7ce5ee567", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.2346341790398583, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:04.923669+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a7a69333-0afc-486c-b475-ecedf3c7d3ab", "result_sha256": "98a95df4e0f13ba96ae1fa47cefef8e6714610f9863e77e30aa6eeb7bee08ffe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.2689813759643584, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:07.052641+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "92c41bfc-ab86-4fd7-be90-e2a89974660a", "result_sha256": "aaf3fa48faaa41ab611d837e4845c672fa28e8e4a92e1cd934552cdeae773626", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 3.2463738380465657, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:38.290253+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e619c132-adc8-4e98-b159-9cab8beadd0f", "result_sha256": "719cfa74babd18396e23eceb5e85a04f5e1cb44cf3990e13aa534532c9486174", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.319595753098838, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:56.153060+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a2f28ba2-5c08-4775-920a-ef57b089dee9", "result_sha256": "ff51c6a8a8b7d09fc4850b49b61c415151914b73a644eeedcef3c89a3bb397c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.5284518400439993, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:50.454487+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "dc963af4-b344-48f9-9f01-48a513d28942", "result_sha256": "89c0e4a9a2d27e88d038f266055968564695dcbaadc3a8230c480504bb322e99", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.0269071869552135, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:29.432219+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0cccbbd7-30e3-489c-9f21-07b7351b3457", "result_sha256": "0424935e89509e5f95a7dc59270e75856b138d24eb8d5d738986ebdcaa3e5688", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.3821549279382452, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:30.484250+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7c70f2c7-77f5-446e-aeaa-2658855730aa", "result_sha256": "bdf887fd7859df4bf6f52af2f48e1be4677789d86577a923a1ef73b71c910cf4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.162842526915483, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:25.313050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "108d01d3-63ec-4dae-a590-c1cbedad8640", "result_sha256": "52f5f3df2da108900c15925d4e1c1537da6f42b71a154d371ce755e939c9e437", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.183015643968247, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:46.138221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "099b3302-714d-4dde-9dc3-35948eed84d3", "result_sha256": "4ad511207b3a9d6ff7c6c9851a93b4172f1d3d4f6146a7c6abeedcdfcc3141b7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.2762355379527435, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:31.195430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "00326f0c-fbb4-4a8d-b0be-bcb59ed79ca3", "result_sha256": "b81a9cea3ac64979b015e48a36543837782763de62da32ba266d0aad9c7f1c35", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.027294861036353, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:25.388167+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "44ca03c7-feab-458b-b911-1c9b84c5362c", "result_sha256": "b628bed1f5bf1d3e12f619e89f4abbd8668be7f21a2a19ef2fb1ba0071ee1a98", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.082425906904973, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:46.432090+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c8c0f2d5-cbb4-4b78-bfe9-4f39c719e743", "result_sha256": "eac596ee615924fd980034941e85da5bd05677fbfe1d05c483e16c9f07a71742", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.8301932819886133, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:59.876965+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "98552743-ba9b-484c-829b-855b0016e045", "result_sha256": "cf30119c37599e126ca06cc94946cb998680e3cf44958804138d3b4bf8b72c14", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.2447374620242044, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.100792+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3a08cf0c-b88a-4a0b-a199-2e6922cb043d", "result_sha256": "782a7b4e8502132fed30a562c6b9fc4a5423f22b6ff3f1b28192a6c4598f6319", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 4.446110393968411, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:47.800568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "826a1cb8-66a6-498a-8147-ddc67cec6d53", "result_sha256": "f279bed41115359a10f1129a4c8d77d421b48d192cb377faa28425e1aa52a5a9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.0686364819994196, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:56.885438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "59f13c50-3d48-482d-91e7-dee00b0300a2", "result_sha256": "a6395e8888d30e490f0591416b2157b2bf8c670aa142c5a4477f6bc7e336cf74", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.105960029992275, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:47.473345+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0cb751c-99c7-4587-b231-7fa81fd620f1", "result_sha256": "35a3b6fc069d7f7c41f59a749eedf868a4816efb8094c37a3379fb83fa58ddfb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9374059529509395, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:12.147312+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "86c31aca-83c1-48f2-b07c-ea4ac56fdb89", "result_sha256": "60420f2b5e51be152c7dbb47cd09bea5d95d242661862fd4af45ab977db18f16", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.0607266270089895, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:01.335234+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "97afc063-2958-4a48-a90c-09273b0aeaa3", "result_sha256": "307854318742b316d9a1b56e89130fcef09b1374e7fd4ed54bf1b5cecd73ac1a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.7723706429824233, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:06.940068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "47d07457-fd39-46ff-bbe9-bf413dea8429", "result_sha256": "5ab05c1c421e6257271cb53068deb5067f1c5b35b2beafe2a544335531b48007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9026185399852693, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:18.906314+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "44599fa9-615e-4820-836b-eddd198e911c", "result_sha256": "1357c046910d1b47e2c6d22db383b3c9bc38eed995e9c3e9f2cff0f74ffa0b16", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9782618869794533, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:11.880719+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "672450ee-0287-4591-a717-2c81a13621d1", "result_sha256": "4699d290679b95c4caa90181b8e5bd696853f84ad7ae477ad2566f4dd5223a25", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.308228017995134, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:54.221983+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a1cc32d7-49e9-43e0-924a-6223c566d0d2", "result_sha256": "82d5d27a42b4487f9837e881b6d4d74159c036cb95c0c52e4625c066e5465f87", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.152431872091256, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:50.358358+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "db63beca-8296-4d77-a272-a06814dbb803", "result_sha256": "fc4e55f01ccba70633db7c13af4416bb7c2c7384b02ec553719a03f768613d64", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 3.5674801979912445, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:25.071518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "49116c95-5c11-4b3d-a5d2-9e6b7cac9910", "result_sha256": "741d954016569583fb8905672dcef2969143694356c4ce6bcc6bcea894e075fe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.468582038069144, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:33.426223+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f6a558c-9808-4d6a-818f-dda7e5f80b0a", "result_sha256": "c3e7646e30ff8256ac88ac7312e2d05122e47641a16fc42a3944fbf27c95f5a4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 14.475530446041375, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:02.227230+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a39ed5ec-1278-42b6-a4fd-2ba4ec379091", "result_sha256": "77eda37566f1ed850e82ed782d366d15855ea4baebe49f4c7b8cfd106787169e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.8594985419185832, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:35.696280+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3b4ff662-ff52-4caf-b430-73787dcbf797", "result_sha256": "f3cdebfb35ce2a454af8baa5c9887618ec708ed2a1d0b0de3d3c251b789078e0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.532094552065246, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:54.190190+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ede08040-7ece-4e55-89ed-6c0881def201", "result_sha256": "b24ffcb515d1c64bf59994de16eb5b24bcb8d1ceabe76ea79b55f95e78dae187", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.056459540966898, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:35.788161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "25fa8bb7-43a3-472e-b150-e6bed6ee1116", "result_sha256": "967b61e3903546e4ca8e92bf6a5e951f40c3dbe35046cf0c21271b33754296f4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.3426973309833556, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:37.292010+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "45a38df1-e0d0-4344-abf9-817087d966ff", "result_sha256": "39f8064f57627f170dcde371cb91bea6cd80b13a3b54215dc660399cf0914393", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.4164606620324776, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:17.151290+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "65f2b1c2-0730-413e-b912-59021df46161", "result_sha256": "c53d5f9a354e1306864bdf14e94ee3374d55f471ae14f6b58c323d25362d9fd6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.964409779990092, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:57.072568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0bca3aa7-a1ad-4f7c-99ea-1fd6b73b5d7b", "result_sha256": "74eb3c61f0d9e9887a0f7dd01712caaa057f41fdecdb9dc47bd56d8929cd506b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.050981136970222, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:42.177958+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "73f20758-aaa2-4e59-a9d0-fbc5f516be39", "result_sha256": "137b5d9bca5be85216356516f3b17a1ab8f6e4b040569274f084b2c849bce6d4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.648924603010528, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:54.466158+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7ddc9d1b-24c8-4a32-9362-ab3e7f414ce6", "result_sha256": "bc86939270a4780ea8cec0d28c2b7662293b29754ee782225e2525164abf5875", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.1378943430026993, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:05.033205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7667f55d-7d59-4b24-a84a-5e74561afd4a", "result_sha256": "c56c38e5dc2fd5d3d55f2786630b5b62d8a5344b4c886cc732559978ce028764", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.8355668989242986, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:20.402687+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3da7c842-e3ac-4e3b-918b-e4cadec3a29d", "result_sha256": "707822dc4e5c79f1fa699cad333304ab4dcc0af81d7054feb29e55fac3e7a192", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.079825217020698, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:35.504926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "433ddebf-25f5-4907-8f89-027ff342a4eb", "result_sha256": "5f591c3c41aaa417ed848b98223787f378b2bdcf975f847330bb6550c9fb3580", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.3435546460095793, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:25.418550+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b18c09a5-f1ad-4566-a919-3f2f8d6c8ac1", "result_sha256": "8e10ce4ee4490121d7862a822fcba2647dde310e8fe4da281958a37167445b69", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9772532409988344, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:23.129058+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ad9d1b88-814e-4be3-9a8e-65a053077ef9", "result_sha256": "b9d49f66138f8fa1c9407bd165e09b6a22e8207b2c99f0fce8f0780961d685f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.1485062709543854, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:36.070464+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4dec9258-0ea6-4f75-8041-e2a8faa7e98e", "result_sha256": "0aee5173b5230da92b8d1c633fce3357b5efde9944b6416eb1398f7e31e7d89e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 3.1574304359965026, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:41.272068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "bf383f1f-96c3-44a7-9fc7-5fbbb62460fb", "result_sha256": "8fc4bdf5281a1e5f6e579339d0ca69d57001bb0e6226e2d5efd8472f9e33053c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.4178943829610944, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:50.653470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ca4661fe-80b0-4085-82a6-8c7685def6f7", "result_sha256": "77ab2328fcd1de4a1e55845aa4528c0135b46c6578a007736dda7cb2358b7c02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.3124748109839857, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:37.466205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "daa21282-d531-4365-a8b3-d7b38f73bb65", "result_sha256": "7dd2cedcd38aba016a99ad0fb3416e7ddb3057f015146db11e189abd0e0c7fc8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 3.043444160022773, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:13.495844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "817e125f-146d-4e53-8733-21252a1dd7e4", "result_sha256": "de13c00dc37948a553987566a296b4cd2fd03876629d4c2ad5b84b3302695613", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.0288546279771253, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:59.109366+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "26d539f9-8838-4c03-b855-22a457e8663e", "result_sha256": "42b6e54051aa53da3f66bf35c1004a4c8d751e1af72065660c0d91760106d93f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 4.120169869042002, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:03.031582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8cad2438-b8bb-40c5-9143-92d138dcc341", "result_sha256": "0bfdf7fd1d471364e2b8b54b0806332428d01c3548f88338c32bcdd355406861", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.8345704849343747, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:16.671459+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "511984b0-1c58-431a-8fd8-c3753a796c0c", "result_sha256": "24dec0e5a4102df8f51d01c515202d537ff9836abf3b6465013e52863859f5ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 4.1829379689879715, "model_name": "claude-opus-5", "output_tokens": 51, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:59.227302+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d1f0f20c-7fcf-4f05-bef7-09933b58de95", "result_sha256": "829f302543cdbf0c7da8edb63bca4183008e6f69bfca6faf464c0e6ee57e9958", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.7771751469699666, "model_name": "claude-opus-5", "output_tokens": 51, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:13.635250+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0308314-6ae6-49aa-a53c-c923c8ab35ff", "result_sha256": "3f9b72b42413fdb56bfda348959299ef0c4e7d5962ee1f79c8f02cb47ebaa4c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0204, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 6.393805840052664, "model_name": "claude-opus-5", "output_tokens": 256, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:46.631916+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0a1be1a-b92b-4ae0-9013-dedec5f4e1cc", "result_sha256": "2895d56937b16b226add4b6bbb4a08e56be12dc076058c47c3b6a1094a559111", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017625, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 3.794309615972452, "model_name": "claude-opus-5", "output_tokens": 145, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:16.263079+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4938798e-1678-4aa1-813a-e52849c3a7c3", "result_sha256": "fc556e8748155d5df06244d2b5c39a1253e084544162875e8f3f320105cce5c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014345, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.296391654992476, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.623667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3466ebd0-7085-4b9a-837f-50999c2788c5", "result_sha256": "eea14fc5ba377578ee1a30c17225fa272b99a5e84cdf00fb972ab3fb2e39d489", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014345, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.4777735819807276, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:32.579347+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d1cd740-4fc1-412a-95a2-8c0753cd3273", "result_sha256": "c0cfc077e70ff896f4185a751d4f97e00570aa8e43385d6c62ed3418d5be438d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01547, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.998409310937859, "model_name": "claude-opus-5", "output_tokens": 56, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:58.787326+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3fedf498-ec50-4276-9b9e-b47953a3a177", "result_sha256": "7770a908667966e0089e1a093dd977f00241b12b27f99856465ce0bd609435b8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014345, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.3617012510076165, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:34.084168+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "988458c4-695f-40df-93c1-b57b893fd3be", "result_sha256": "a9e38f34c6192a5ca73d69d4987ed3bdacc8c1c6ccf7f33e97ed4db039657434", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015545, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 3.3209629940101877, "model_name": "claude-opus-5", "output_tokens": 59, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:43.263107+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7e8ee28b-00d7-49bb-86a7-e517c68a106c", "result_sha256": "32eebd4ddbc2df4ccb47df1b2d408ecbb7b2e57e8d7d14e02f078a591d1d8ebf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016945, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 3.0609125760383904, "model_name": "claude-opus-5", "output_tokens": 115, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:39.528301+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "009ecb4e-81de-4b52-b150-9ab3d9ef71f7", "result_sha256": "12d050217cd58fd82817d408e2f569b2eff2828425d47eafb49ad82437cc86d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01552, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.3478611019672826, "model_name": "claude-opus-5", "output_tokens": 58, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:52.197968+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a43630b-a8f8-46df-a1ee-009198cb1fe4", "result_sha256": "398eff6c1b6210cba87ed333049bb8350eff173a13cc2d0fa0be89582c07c5ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015495, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.772365016047843, "model_name": "claude-opus-5", "output_tokens": 57, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:35.452730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f4b2947c-bd03-4a52-97d9-e1aafeff33a7", "result_sha256": "1d72be840fad58d2aa54acb32115743b592f2bcf0237835edc2982b5980fc12a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01637, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 3.357606149977073, "model_name": "claude-opus-5", "output_tokens": 92, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:03.475559+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a992ee6-beb0-42eb-b2d1-65e816cff801", "result_sha256": "10ebd5c222de35bd9695b7d707530d5280199330539bbdc47c2123a3a3400891", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015995, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.937611851026304, "model_name": "claude-opus-5", "output_tokens": 77, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:54.445851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4519996f-379c-4fa0-81fc-d27872cc3e31", "result_sha256": "3b3a384b7fe3c365be9d7182353453a650108ed8a5d98dd66cf9394ec192dc5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016345, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.9129479459952563, "model_name": "claude-opus-5", "output_tokens": 91, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:42.218646+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "214f5ba2-5c12-4f60-946a-18b57d2b6482", "result_sha256": "d99b5aec487cab7afafa9fe4c7744bdc76e9174651d5ff7d6cdc25cf608f9e32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016445, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.8408228179905564, "model_name": "claude-opus-5", "output_tokens": 95, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:46.930291+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b1dc0731-4b15-49cb-b499-33170961727b", "result_sha256": "36edcafd973a6f5715126bc03ca4f567d23367adfdc11ca8cc6b6169df10d626", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.244080572971143, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:05.339174+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6379cb7e-1ba8-4e21-a928-99e669279382", "result_sha256": "86543015cc292a9063ea41e6028e92a02e72db0f8c662bd2a06666afd64579f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.139325671014376, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:31.941487+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "99ba2738-8cd8-4026-bb6b-701ab2c816b2", "result_sha256": "9a77fae0cf44c5b92bdf12f981208d4b00eb0c86d79f834fbd0f985610a1bba2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.0180706990649924, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:49.531284+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a3af3d34-28a4-4370-847d-22decd25c526", "result_sha256": "1dc98848c26646d85fdb75563b58e6f5eba7d0c9c8822c30f0bc25ee4741ff47", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.6977497891057283, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:27.033801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "21384e33-1e5a-49b4-9439-7a02cb4ded35", "result_sha256": "a42504d388af83cf162528cca41902d9adb41c821ff8ebbe214cefbd0d786eba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.4617198299383745, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:01.867415+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a9e26633-d5f1-4b53-bbb1-76cf18dbeeac", "result_sha256": "88c209d9369b3a8b44746f0da5c8a1f9d8cba81f0ef88c9bd287e6b355e72bbc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.060147244017571, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:49.853454+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6798f673-20b2-4255-8556-9fe81a955998", "result_sha256": "ecc832fbad2ddfb79ef1b090b59582a3d05d87b2b052573d2f292bf1712e3caa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 10.261918324045837, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:25.819686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9491331f-ea0e-4884-860c-55e8ffef01a3", "result_sha256": "39e47a8e5f44533fa79311988086e99435bb8ffbd981dd5073c4e40eda18b45b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.085148103069514, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:05.258216+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ba270137-aac0-4075-909e-8808c9a3383b", "result_sha256": "735ce004e8f6267a82f4a6ce419ec316c649f4c35b734db9f5749ec423ba32f3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.0345410130685195, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:48.536981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "91ea5ecd-3f58-4c3f-821c-c34d484aefff", "result_sha256": "b8f5d191af8973c37d12c97bbe9fb939d75a36ddfa57b78df5a229bbf9fa6156", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.3158883000724018, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:49.006841+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d8c199e-e148-465a-ba70-48211e9b3c5f", "result_sha256": "2cbfb74f416d604dde9df48b655d0ad723fed6303d293a97c0474bd70e5092d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.134305367944762, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:53.861867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-opus-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "fe170910-5580-4abc-9040-775ce9d17cd7", "result_sha256": "0de84dc0b3425ff6a9ed0f7184d54428314985023eeadbd6faae17062659f1b5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01419, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 2.097016524989158, "model_name": "claude-opus-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:41.501267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b10b72c-bc80-4372-9034-0aa39aa407c9", "result_sha256": "2b948a9c55501905264da820f2cdc57b01307bae97762965eab04d9a9190b47f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.9855695770820603, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:17.679441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9d01ecc8-348e-4eb5-ac8f-7f14e87ea3aa", "result_sha256": "cb9f5edb952b5bf8d338963cec6e223aa5d48582c590f2bce17db0037b2308ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.792643403983675, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:13.957466+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9625da99-cdd3-4a96-83ad-303340cfcc82", "result_sha256": "5595cd94c3a701201d50d910866fa4b36fc3e9c749e12ca8cb9e7109ac067766", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.9874364970019087, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:07.976303+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9954d8da-2382-402c-977d-6e14df3b70bc", "result_sha256": "0059735c4f161ca3c95e37d835c0e1501c2005af093f51f9e70777fe7bc73e9d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.8498593580443412, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:10.561155+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f35a800-5e4f-43ad-9a80-308e5cf295d1", "result_sha256": "d72674f2fb6ab5198baa33426d32183574346b3bbf6d4027af0f0bd355de4a54", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006398, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.410622815019451, "model_name": "claude-sonnet-5", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:37.296988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "22da910d-ed01-45c3-a5f9-34b1ed2e299c", "result_sha256": "3dfc9d560df43919cef68a09ae8b54d59f2d96123ed20bbb66ccd47b336cec9a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.806738344952464, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:52.236220+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "598d8710-5ac0-4dbc-9f5d-39ff082e7ad9", "result_sha256": "fa378a210420ba07c2d23b3d02795dfbab6fd1b1851aed2d702108b1c2693c04", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006498, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.2913989609805867, "model_name": "claude-sonnet-5", "output_tokens": 85, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:39.171953+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "fd9a5d56-1ca8-4baa-b462-9c540712e8a2", "result_sha256": "83835642342840ba732b1fe40bb4b8b732feb00f4c09098a29899050b0cb07a3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006228, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.555660932091996, "model_name": "claude-sonnet-5", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:22.189290+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0f31af88-3be1-4ff8-9123-3b5f9f4f86d8", "result_sha256": "b40d12a2fa534e73d0454762830a497092ff9d7e0d786272572dc22a0f0d764a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006488, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.2246478599263355, "model_name": "claude-sonnet-5", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:29.301614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "66d560e1-4e85-4888-9391-1113f2fa889a", "result_sha256": "ffbf865f145edcbb465d0c07704072769eb1f1ee3b43939bc23c330f20778863", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.522202803986147, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:52.344430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b43822bd-8dde-42c4-9b3b-36223782642c", "result_sha256": "c631a41ede7c2f05a2029e08bbf8a196b77e1b606da479ac49cecf41484faf78", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006448, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 2.7233662490034476, "model_name": "claude-sonnet-5", "output_tokens": 80, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:28.712552+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "95c98eca-7fa9-4aa7-b6c0-83ab44ae03ef", "result_sha256": "5e0dc05b2302fccaa3fc062e19b37922c5d3f505e16f149d36b72b191282fdff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005758, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2824, "key_errors": null, "latency_sec": 1.2581240859581158, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:48.992059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9afbaf3-cfa7-4e96-bc85-380abf481597", "result_sha256": "9667483fe52f6cfc4716a014e02f21cde837d9d87ec89b80a17ba513947f2146", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.48488995898515, "model_name": "claude-sonnet-5", "output_tokens": 110, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:56.623216+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d67d99c2-e7ff-4767-baf7-35377080d05a", "result_sha256": "61647744798fdbe8057bd7c0f8dbebc4d106b9eb58c8d8df243410eaee28fd3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00649, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.9516219070646912, "model_name": "claude-sonnet-5", "output_tokens": 59, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:59.938082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e97cd292-f18e-454c-8a03-31890b02d083", "result_sha256": "d9e11d8843c4cdf953d0e399050d128e4fa8968c04624e8fcc236cb2de2b949a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.8090167680056766, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:59.209161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "887f29ec-4c89-436c-b545-061a9039ea7f", "result_sha256": "32f268896726bb83203ecf17320302232796aa0f3af3f93873051107e3fef61a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.6277407599845901, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:04.402073+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "efc748bd-0e80-4b00-b750-4c5ab2f9a804", "result_sha256": "2c9a6de5da57b11a262acd587b7956e79b339eda55c3bf63a4681c8d09eaaef4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00859, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 4.297187029966153, "model_name": "claude-sonnet-5", "output_tokens": 269, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:05.973129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "be2d9ad5-89e0-4df7-b228-5650b0dda103", "result_sha256": "fa177e99f40302f979e0ead505313be50a6722301f7247622b616a4c9673566b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.047519780928269, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:09.804821+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d297a128-e10a-45e6-8dba-432df28b34d1", "result_sha256": "4eccd5f41e865e315d5ae950e0bded3b37a369a4d229d5fc4e1be4ed967290d5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00884, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 4.751874592038803, "model_name": "claude-sonnet-5", "output_tokens": 294, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:44.520481+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f491e8d3-47e8-48f8-b45a-22895aa69d9c", "result_sha256": "7065c2e0c7f40787b1780f5d3402a339cf83e9356629cc6500a6e626e41042e7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.170659836032428, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:41.651450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5159b372-c4b6-49bb-ae85-6d76ba019a3f", "result_sha256": "29d1b2785c2388eaede4bc072e6e1dde4fe6cb44baa008c8a2899daf213fc8cc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.3900387639878318, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:10:01.974346+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a3ea76e-0a9d-49e6-9f91-051e07a24e9b", "result_sha256": "6fa6853d1163598a876c8cc8985802cde60013ab43d499f3e82cf91549dc903c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00663, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.727523957961239, "model_name": "claude-sonnet-5", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:22.197482+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3c45010-4fba-45e5-8837-9d9d774bfe64", "result_sha256": "0b8be570143bd862972c87dc27bcf488cabfd6d14678f9d9fb8f1bb75888d6c3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.6735792710678652, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:05.992486+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "51435605-92bf-4d77-99a1-1e8419c9ad3e", "result_sha256": "cc3f35d8e01002136438f8bc44ceef02771dbbcd4d0ddc511a9eb2ca4c438b6d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00778, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 3.901031270972453, "model_name": "claude-sonnet-5", "output_tokens": 188, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:51.354555+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d563dcd0-82dc-46e5-815b-547b693543a7", "result_sha256": "8d9cecf2139b0a0a3a720557910ee1f92b2a3b8333d29879e1440da74c4182fd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 2.0001559000229463, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:47.775951+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c0d74690-9ac9-4efb-b3a9-66d5b249a6e6", "result_sha256": "6e21443ff905421f24c6cf68084dc779f954a3b7f12f98968302e75710fe49d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.7694852019194514, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:00.951977+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1840c76a-8eaf-464d-ae87-dbd62d8507ec", "result_sha256": "999738165d4bca85da6feace30b9fa6baa88979f6820f7d60115b0c8f1370173", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.7306144459871575, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:56.173616+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1930ae3f-ee91-4b00-b31a-4b5fb0038016", "result_sha256": "986e0a5304d2bdf8f953714192955e448dd040bd2f507dd71b087b39f6f7158f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00601, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2950, "key_errors": null, "latency_sec": 1.6535218720091507, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:42.667058+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0d2da944-bb8e-46a2-b1dd-6a0c76621f74", "result_sha256": "e9abdfdce968af72c45f2499d4e4fb20f9c0ce0d7bcfafba6ddca450dd467e66", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.8654006279539317, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:38.737788+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "942a712f-2f79-41c3-b85d-50ca325503a1", "result_sha256": "5d2b0a643bc19d49c7330cd5be778379178aac4f22627887a92ea9e4847b064d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.8466897730249912, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:09.364909+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5d2d976d-5a3e-4ef9-bd3f-41c61210e622", "result_sha256": "915214bc9e98be81d9922255440d1e2c419c1489a4ad687a6cc0a579998602e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.9150350649142638, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:28.103725+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0522def3-8fee-4915-a524-d82bc95bb13a", "result_sha256": "b26093178f9f030d747227a45cabe90991e8905f1cb8d9f25defb0497d23c8c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 2.222823846968822, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:23.281594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a79b999-e82d-440c-8e20-b57762fb21a4", "result_sha256": "a9133f42fef61a8f21fe28078b37982cee9f59370cb1546f3bc5fa9f54a6f306", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.7446646600728855, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:31.944286+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "29d857c3-3da9-4e18-89c7-9819e0b919af", "result_sha256": "da3dc19bf2eab2599888d529063c2ab04e93e3f9c77fc2575ff9803b383afb76", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.8739139729877934, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:58.635795+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce5cf5ea-cc98-4030-8dbc-3ee9f218a90f", "result_sha256": "1f33c9031b5702e34b4703f0107fb31c7a15247bb966a3072b44a9dd28b910cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.7761452429695055, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:40.518760+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1335878e-14ec-4b1b-b75b-c231e17f76ce", "result_sha256": "c5c8276a9d14cee4ecd763187a61503f0de5137900fa3b2cbadf4d4bf9226fbf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005684, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2787, "key_errors": null, "latency_sec": 1.9402421499835327, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:14.550772+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "85d3d8b8-640b-4b2d-bdd9-1bf5184afa66", "result_sha256": "1e435b9af717828c885890a5708452231cc6c05400354a6203c902eee6789ec2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006598, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.9707984320120886, "model_name": "claude-sonnet-5", "output_tokens": 73, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:08:56.261983+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4750cfd5-9258-46ee-9f6a-8a641d1a4ad7", "result_sha256": "f8cfeaed43b4ab48ba4ca15477ccabeea8d464bb8b6e8eb3881a60a6dcbdeaf4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016638, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 13.729338833945803, "model_name": "claude-sonnet-5", "output_tokens": 1077, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T12:09:48.293742+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c5e1a825-1131-4477-bc70-a65f1dc6f0d5", "result_sha256": "021f8d4c2e1ef56952276c6064114f62f2274e360578f7b5fc994ab24c76e386", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006268, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.050002687959932, "model_name": "claude-sonnet-5", "output_tokens": 40, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:54.901732+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "fdc57565-81da-406b-95b1-e9a4c58deeeb", "result_sha256": "acc95e4a46fe5e77da6d415d44a02da225351b58dbf2443c25b7c2c33258051c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005978, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 1.8207180729368702, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "F" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:13:54.590273+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "45d0432e-2af1-4695-9536-c807521bbdd1", "result_sha256": "e2d4989bb06854ae6cd4728953c7c9d8b51b1cbc475b9a11f6c1dfb67566161a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006858, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.502908758004196, "model_name": "claude-sonnet-5", "output_tokens": 99, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:13.995844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "fa918a4c-4bf2-4deb-9bf9-50efbc754a69", "result_sha256": "a58e7cb0a69323bcff9ab3c310fb705ca928bb21fc45872fe24b2a7886c81a5f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.018718, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 15.27232546696905, "model_name": "claude-sonnet-5", "output_tokens": 1285, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:43.577360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "40ff78ae-c6ee-493a-b498-25f14eb9557e", "result_sha256": "8371834c6a1b085791ff25aca9223e4316ab0d6aeb9d33f31f71c02a1030be30", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007138, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.438010416110046, "model_name": "claude-sonnet-5", "output_tokens": 127, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:24.156897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4803db15-af7d-4b27-b025-3a9f99c17962", "result_sha256": "7e31e16cca150e26df81e3a84c713b3ea5afa618aeed3ca1143536988720e62a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007158, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.6904853919986635, "model_name": "claude-sonnet-5", "output_tokens": 129, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:15.994208+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a20fd5f3-c315-43b0-9d96-2530d55e0e69", "result_sha256": "11e11976bf50cd232b996c14e88e650693c7473be60b17c8d7dae673322300ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006668, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.42094891902525, "model_name": "claude-sonnet-5", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:21.771565+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd0311ae-28b3-47f8-8f54-e50202eeab9c", "result_sha256": "d7eecefe1036f290720a073a621af0f0c33c1940cd78c6cc3849950557586367", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008778, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 4.1550521169556305, "model_name": "claude-sonnet-5", "output_tokens": 291, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "anthropic", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:22.016524+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2cf6d30f-87ed-4078-a8a8-bbc054cb2d10", "result_sha256": "3f3115bc99fafdd2fff6d6ecc09f8c99082e681097e3e822986176609d08cba9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006628, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.271789849968627, "model_name": "claude-sonnet-5", "output_tokens": 76, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:49.054795+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7592847c-348d-4ed0-9760-ac2a7582f284", "result_sha256": "635549cad2d50500d36b44a0552d332240e33d3f38c6717fe153e21784d7cd15", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008228, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 4.403347961953841, "model_name": "claude-sonnet-5", "output_tokens": 236, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T12:09:22.390776+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "89115849-4a5a-4390-a7e8-ef439f67cdd9", "result_sha256": "b01261173956a016b64655913cbb3ac232f3a98ab134b56139d88d18917884f8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006528, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.3348386849975213, "model_name": "claude-sonnet-5", "output_tokens": 66, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T12:09:45.598262+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5cd9ecea-d5b5-4d34-a018-853b304428d3", "result_sha256": "717b945c61d9a3219865fd4fb7f5bc3b569ab3e7881fb4c7ca9aa4c6ef634490", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015628, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 11.443716884008609, "model_name": "claude-sonnet-5", "output_tokens": 976, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:51.834686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "fdd95ea1-8114-49b0-85dd-c3e88677b8fa", "result_sha256": "65e33a6a0d86c594b0de5eba0e17b221720e8709a5e783e22a63b0299e462375", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007388, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.7955728280358016, "model_name": "claude-sonnet-5", "output_tokens": 152, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:39.969403+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5116c71b-f763-46f4-8a5b-c1ffd3f17577", "result_sha256": "c8db11588e7e74f7a0ef4000b1df795ab62435fd4fb83b73ba2e0afe2d2cb45b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007898, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.3312806920148432, "model_name": "claude-sonnet-5", "output_tokens": 203, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:33.674307+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f46e941c-c806-4c91-9680-eb227df52b3c", "result_sha256": "b86e15e76220650ce53b718759d3ec469355fee0922cae2492dd9f75e610a49c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008038, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.652394859935157, "model_name": "claude-sonnet-5", "output_tokens": 217, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:23:40.702822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b38b1754-e28f-4d2b-8ea5-2d1272c65145", "result_sha256": "e53d178923f8f7a853a75c35fba8baec2befc07fadf71e19216c15dcbae5d879", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009698, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 4.821384762064554, "model_name": "claude-sonnet-5", "output_tokens": 383, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:26.550357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d1154ed-81ba-4b86-b3ce-70546202ebac", "result_sha256": "b914b252a868dd6bb784034f8499be7a987f618d3742df842a43ad39a27f3201", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008108, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.748614507028833, "model_name": "claude-sonnet-5", "output_tokens": 224, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "anthropic", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:03.237645+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f3a8b09d-4ac2-4ff9-9cbe-ec9fb5b524ac", "result_sha256": "ac74ad31637fc80aa7d1bb42333deffd346ed0f3f7b665579e3f108218cc9d68", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008218, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.5538153080269694, "model_name": "claude-sonnet-5", "output_tokens": 235, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:02.487399+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0b4c044c-7c8a-463d-ab5a-e1c12715c6af", "result_sha256": "11b7a5504f4070455b04521de657d4c9fcbfc405536c11066e1e68e85541a286", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007218, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.841779852984473, "model_name": "claude-sonnet-5", "output_tokens": 135, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:59.342645+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ba964682-eb47-4ce6-bf4f-348f7c23772f", "result_sha256": "78d7ef0b9b94558de3e165e44f5bcc8b73865fc82e9aba5db1ac85e84f8e475c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011578, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 7.602495500002988, "model_name": "claude-sonnet-5", "output_tokens": 571, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:11.823168+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0df9d83f-47e1-48d9-bba8-b6c626d0e51e", "result_sha256": "30d99ce5ae03f7b922a70c0ea3dc029b9fbc97f9998ed9865999a9752b315321", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007178, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 2.8761821819934994, "model_name": "claude-sonnet-5", "output_tokens": 131, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:33.176425+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "956220b3-2f2b-4d94-874b-6c6f6817d54e", "result_sha256": "5f8084c65f9453fc370d33ddea5f94424ce9217c1860806f78746728eaef5edb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007338, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2934, "key_errors": null, "latency_sec": 3.6746733649633825, "model_name": "claude-sonnet-5", "output_tokens": 147, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:54.484832+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "60ad7b77-8f58-47f3-8284-250cb65c5010", "result_sha256": "4114179bf0d87394cf5d537faa686768e51049a63edf3166499f1bcb1bd5fd3d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006518, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.8477774289203808, "model_name": "claude-sonnet-5", "output_tokens": 90, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:08.272382+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7cfc317a-dae5-4524-8db7-1dbfdfba30e3", "result_sha256": "ab49dda3a1ce4e06fec365a7191768dc5f8bad9df555365d81c87578f64554ce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006388, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.564891543937847, "model_name": "claude-sonnet-5", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:58.142921+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "82db7978-6906-4fbf-91d0-e09eb0f65307", "result_sha256": "fff1a5a141cab5388e442c40a7dce48233ca6242e9a630e289be0954b8f38ea6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006048, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 3.250611466006376, "model_name": "claude-sonnet-5", "output_tokens": 43, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:11.379413+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c1d53ce-12c7-40dd-9434-07608b9e5375", "result_sha256": "1b2be221df43c6d9ed74c785a7047e2187fd991cc22ae2d56fb4d56f7c239120", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.18851424602326, "model_name": "claude-sonnet-5", "output_tokens": 75, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:51.425390+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a36a5b39-b771-4953-8fb8-6a4419cd0f37", "result_sha256": "58e6006c3fb483faf507867816973d16e65ca39d86919039cdda0982c738c608", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006508, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.1632418850203976, "model_name": "claude-sonnet-5", "output_tokens": 89, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:58.007627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "72f8336a-28a3-4501-a7fe-bc4f4bdac541", "result_sha256": "463e2897b91b2366252980ff6e3d88cdc00c8f221c64034e7b539d3f244edc7d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006508, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.1591046950779855, "model_name": "claude-sonnet-5", "output_tokens": 89, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:58.786631+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "fb66f427-1cbe-4f25-91a9-a8722f0e07cb", "result_sha256": "f0a4a2a3f586c3c8b686cd8c6f36dee643ca505fd0dbd2b528c8625df4624400", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005728, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 1.7311993250623345, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:17.169736+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "45a838c1-f760-472c-a76d-a432be53acd5", "result_sha256": "56f831b5f1153f99039eddc92b29582873cda8fc2a8f5d89334414de6d3d2e8e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006478, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.2406098790233955, "model_name": "claude-sonnet-5", "output_tokens": 86, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:35.681107+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4acdd719-dbbf-4520-a91a-a197febe72b6", "result_sha256": "1c653810dbcad3add7243b9f7d14b874234bc041fec31fec3617ae3cad1e0444", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006018, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 1.5136526779970154, "model_name": "claude-sonnet-5", "output_tokens": 40, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:05.483909+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8ef22892-4f9f-4234-af53-de499fbdd80a", "result_sha256": "1775b484cee6abd976724679a031ad45ece21b2ba10c9b13f72775922b88e378", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.3072714139707386, "model_name": "claude-sonnet-5", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:32.602860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "67c8ea38-ba9f-45da-a080-6c8f4f5d7839", "result_sha256": "f9bcf4ef92427600e981cf2963bc886606dbdc9be7f566bd9c1806696d9be684", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006018, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.020336564979516, "model_name": "claude-sonnet-5", "output_tokens": 40, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:17.044786+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "50f85f42-bf9c-4c37-9855-849fe59287d6", "result_sha256": "e0c22d47f539c6a5100c3316ec7e57bb51c53651c8d57f516c6dde639a9eecd7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006348, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2809, "key_errors": null, "latency_sec": 2.835161685012281, "model_name": "claude-sonnet-5", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:49.907886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f41473ab-fc46-4368-a2ea-f3df79349667", "result_sha256": "741934c42b7bc1b7fc994fd7b82fa288480a0372c0ceb9dd86474bb5b5565252", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.5628063899930567, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:08.185335+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7477ab4c-359c-492b-9537-94f4005fca00", "result_sha256": "ab9a7e18eba7a47a3b6329e4d5e59650a25c0502d1eb61ed666e3df22303bed3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9363128000404686, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:23.021006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a05dbaf-c092-4d99-a314-b5b3b2170f70", "result_sha256": "002b913687a934bd1a9ac6fde7a4bfaab2648ccfddce0148241f82f9fdad4dd8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.0464949948946014, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:39.425397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a91c24ac-62b1-427f-92bb-95eccf628bbb", "result_sha256": "5e986b98991fe5f5a1346273dc8b63dbc7df97da91cd9b4418f6d5675b715d02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8197744010249153, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:40.558396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "22505c9a-6bad-45df-a520-eff3ccfb5743", "result_sha256": "c6920532df2bb52dffa06b0bdb165d31dbb9052cd606e4f6ab3e123a57777b20", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.177576476940885, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:45.779966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3864e92a-d07d-4c56-b6e5-aae8da9bc4ab", "result_sha256": "7cd169349c6b0c3c2d0d35f0c6f2f63d32562c182cd266f7b5fb560d8daa23e2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9978810040047392, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:41.764810+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a863b8e2-b057-4a64-864b-04c2bfcdaac0", "result_sha256": "0934a5927888c95aa1b090326ce9de45682db261bde3400a5291015543221d80", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8097283049719408, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:54.008403+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e88f6983-d51a-4e9a-94c5-c092644f4fe3", "result_sha256": "823f66e3b219540507aca6b03fa40b9762862dd99622c775afa05f890ee7a17b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9203671929426491, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:58.250032+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ab5ca286-a6b1-4739-8890-65abe7b5285c", "result_sha256": "1c92e6b29fd969646fe89b2ec9d541ef42d748b69444d1abe8534b11bdd26cff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.738941483083181, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:36.227991+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "715d51ad-ded9-42aa-86f4-d6b313d7e79d", "result_sha256": "abc8833783f620501cd0158cb11ecb6617b182e2ce1b84a024ec86ecd012ff90", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.7311281370930374, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:47.263913+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5d3ccc1a-ce12-4281-a670-ae1350d94ad3", "result_sha256": "0bad3f27c2e18edd8bda925084c72f9b58b5eeb0ed1634df7906a4eeea65e238", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8819109620526433, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:32.277621+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "188337aa-8c86-492d-bf3b-a2c69b2d7f58", "result_sha256": "2f50ee9f8147bc1d78b8d5ce2873cdf6df2e330d52f181e810a47037cecbe821", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.2151726440060884, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:08.365531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd2ec0ce-a900-43bf-9208-cd42ee7d9ac4", "result_sha256": "560f559a86129d034f31bd71416302876fe0a6824e7e09fcfc2f3c824a6a848c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006792, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.805858619045466, "model_name": "claude-sonnet-5", "output_tokens": 114, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:03.501678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd7e1ac1-5e7f-4724-a0c0-1553184cd5d0", "result_sha256": "363ed6901270cec90ed1a6fba7c3cf67fe503a4d2ae8f2d4a7c83eccff55c4d6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006372, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.264330642996356, "model_name": "claude-sonnet-5", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:12.557697+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "46a53a81-8d83-4b20-b149-75f56185f810", "result_sha256": "a2902b851208b9e25a7459b74c8bf250f167ff0c75986669d00150d8810e7c8f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006942, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.7524967619683594, "model_name": "claude-sonnet-5", "output_tokens": 129, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:12.162229+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3bf2338-1b9b-41f9-9610-604afef74308", "result_sha256": "2dc3b79280fadc21742caab97a74b64d8baa7161c18e941512325ad9a491ed96", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006362, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.7251612109830603, "model_name": "claude-sonnet-5", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:27.714914+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f269d8cf-60eb-47bd-a973-7915f253fefe", "result_sha256": "0a5521065b662788618f9607f57ef5cef3fe55d4370e11b7b77eeeb9e553d723", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006582, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.9724495799746364, "model_name": "claude-sonnet-5", "output_tokens": 93, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:26.619042+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "203dcc9c-729e-4b52-8900-f5b62c769cd0", "result_sha256": "55d5da81aef5ccb4b43b8c4fe6980c4717e01a3190630f14b6f95a387315efdb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006882, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.5568224999587983, "model_name": "claude-sonnet-5", "output_tokens": 123, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:38.136046+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e87a4efc-0390-4b92-a40a-2ca97825b3d8", "result_sha256": "5b15551d315a8aeb16369b5f75fe64c02d88def20a6b26b692466b18890b1f0c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006552, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.4702348400605842, "model_name": "claude-sonnet-5", "output_tokens": 90, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:57.539130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e9fc0af-2220-4e2a-9803-50cfd21892f8", "result_sha256": "b4528e0f0ce9d569409d287e2dec90480af21fb5f7016c1b0c8868e51844e589", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006662, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.438442953978665, "model_name": "claude-sonnet-5", "output_tokens": 101, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:24.703179+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2e1fb529-9556-473c-92b4-dc91a2fc141f", "result_sha256": "5cf017c15102f0ee7c099e431a9f471aa87b1383660f5176f768eab7a3b88f06", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006952, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.81479960703291, "model_name": "claude-sonnet-5", "output_tokens": 130, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:36.629555+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e597251b-5f41-48ec-a0b7-719a661b6317", "result_sha256": "de23abec4ab5de02aed96f1e37343fbd174abb02b7e02059edfe52283307cb50", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006432, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 5.360130065935664, "model_name": "claude-sonnet-5", "output_tokens": 78, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:17.639445+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7abe6d1f-c4a4-4b43-a8df-03f453e70c36", "result_sha256": "9063d8473d7e106447a291358945335ca86c125d46295430738147e76b2b7fba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006532, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.1971416290616617, "model_name": "claude-sonnet-5", "output_tokens": 88, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:06.393969+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8105de83-3bd0-4bf0-b5d5-bca0ae9a83b8", "result_sha256": "594ac22423a1fe5d318893d62bb2dad4299def06d84df579be93a47e42102d0d", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.006242, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.0450759700033814, "model_name": "claude-sonnet-5", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:18.576390+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9c3ac49-3bbc-49f6-b64a-d82100e86b68", "result_sha256": "5d2bb6830c7604ee6d3aa46638788ae6691a8d05c66a6eefa6f9b10047374a1e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.759688265970908, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:02.633883+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "de893c91-9516-42f4-b7c0-891b7c3d212a", "result_sha256": "900d1e80208142be088f9721d69738e45707c5d7d5bdbabd0dd214f9ce1ce5e0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8085565499495715, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:10.122560+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2df233b3-5d0e-496c-8c18-133cc944dd91", "result_sha256": "e38e2f6cf8e2629d3428dcbbac27bc8bccc7e0a38b0bac2dae067bcbe1290e43", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.1496509860735387, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:00.176267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "257963dd-c68d-498d-9788-39f3c5c30804", "result_sha256": "ae7a9a91975adc44486bd3b555dd319956995cf5a0652b97c5248abdd1a51f4e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.2412164049455896, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:52.123401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "52767f9f-e33e-4b78-986c-30fdd5193021", "result_sha256": "037a18085f63dbe84ee249c74e42c90abd79899623e7879175bc0b6e86120bba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.3773766789818183, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:29.794275+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "61ddf655-a2cd-4b0c-8087-b0ca41379ba6", "result_sha256": "f87adfec82038aba6be54234889317c3865fd1944b78da9cd058225dd7fe813e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.7404538140399382, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:43.331443+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "91c53668-bafb-4348-8dc2-a048d541c40c", "result_sha256": "206cea2e15be736f2e9dbfc7adb4f5a34a6aa481988398b6a79a9f743a2d7aaa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.1847796459915116, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:49.917175+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "45785f9e-9f9e-4aa9-a2dc-4175c6b887c5", "result_sha256": "91f38a68ae05726ddc007e58a4cf6e8086895c4dd75217eada79ad5b57ddddac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.6587127139791846, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:03.774439+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ee9ddcf7-9de4-4a92-8686-709745c8f436", "result_sha256": "b636d5d24bdfeb20dfcbcf1403fc79f4f663071bd67594a5bcfd86ca73ff0746", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8969414910534397, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:27.745694+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f349d79-9730-4c86-a32e-f46c3805064c", "result_sha256": "b32ad2f4ade5af92e4ad0e29855099d5a5f3ca0d5fce126843500623b83a4a7d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.8525107860332355, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:25.128153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b347d70e-d024-4f91-a536-e20cf1c3f836", "result_sha256": "c0d0419007976daceb9e816eb866079852553096f01d88ce5586c375cb492dfe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 1.9335799060063437, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:51.619276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "38faf285-4f9a-48a7-9d21-c1866a35cc27", "result_sha256": "7f3ec7d1d63dedc4b137315415033c1d89c1de02773a3bdf41093e5f1515ef8d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2786, "key_errors": null, "latency_sec": 2.0282433460233733, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:50.234373+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b6713524-f018-4fd1-bac6-a076fb7f36d7", "result_sha256": "d57248b58785356eee95c7660d454b866a29cbc74e91542824e3d0d2a26927ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.7750401130178943, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:53.731535+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3003ada6-16a9-4b72-b80d-5b8e0ca70d22", "result_sha256": "56797e090cb1fc59cb20e089b2429670dc3e46d1475782e6c7e92a06a938a41d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.7354092119494453, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:08.203366+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "931cf35d-f0d6-4856-ba46-4cfd353ae669", "result_sha256": "a10cf93157f761a6d7264c6c6ced86667e237e6c44e917345dd94fc7b69b0966", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006126, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.891954972059466, "model_name": "claude-sonnet-5", "output_tokens": 50, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:43.844545+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0765a1d0-71cf-470c-8f75-588416fe9127", "result_sha256": "02cdaf2e67e8a4a2563b53ab48313a13cb122b7a0692714992af92bfe6ba35b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006276, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 3.9754105739993975, "model_name": "claude-sonnet-5", "output_tokens": 65, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:53.471773+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "56188c3d-11fa-4195-a5e5-7db32e609131", "result_sha256": "5ca1914f442143f2acc6d36cf2575c666b0c199cf3b82ac1fa3dac87926c1186", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006376, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.4541401440510526, "model_name": "claude-sonnet-5", "output_tokens": 75, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:25.442396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "028d3d84-75a4-4330-9a9d-18e7a2a04b29", "result_sha256": "8176153bfb9aa9e195a536bd3681cdad1fcf50ae96b19782da123d2c2a7225b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006296, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.302712721982971, "model_name": "claude-sonnet-5", "output_tokens": 67, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:17.419883+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7bc8f142-b2a8-4813-ac2d-6918becd3413", "result_sha256": "6f469314673f42b81f1b1b52a244eb07a67c19fa176b02c02cd522ba4b2af753", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006306, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.7572364759398624, "model_name": "claude-sonnet-5", "output_tokens": 68, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:51.152841+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3a9532a3-6a67-4e41-b05c-6f4bfc6c0dc0", "result_sha256": "8a69d5683617a10d4a4d6f44196bb0e6791e5c3f66a4fdbb869f12e57591e6a5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006816, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 3.207823913078755, "model_name": "claude-sonnet-5", "output_tokens": 119, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:40.749337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "40edad89-62cb-4001-8724-0cdfe6a3cb9f", "result_sha256": "841c359b65f5060e875443e0a63443f9f04f8a9e9f54021ee26ef6c90a315bda", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.816521006054245, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:54.862479+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "68e9327c-965e-4e2c-8c23-a3fc1ccbd6fd", "result_sha256": "64c3b1464853c67f510bc7355b6f1f6dce92865f9a32e11dfc052cc24e9c86fe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 2.1839894120348617, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:04.055103+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "33adb08a-9af6-4a43-879e-2321f0f081ba", "result_sha256": "8e029ba76625ddb42258c7ab145eeb1338d6301aaf24b4e49dcfaebe0590287b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.9309732450637966, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:25.696276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "792c4e8f-fcfa-4818-91d0-0562c47c562e", "result_sha256": "0a6a58fca551ac26b5853ee57b9e947d950f092e0e506736633005f0a2529839", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005736, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2813, "key_errors": null, "latency_sec": 1.7709152249153703, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:14.575447+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6c877044-bc3a-4e29-8f4c-85dcbf76a74b", "result_sha256": "a37314a67cad2f422dbecf721b9bb60b9347027059cff5d3776bca79316eeba1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.1226355229737237, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:41.763667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6ceea81e-e765-408d-97f6-ed30ec93ffd3", "result_sha256": "9932dd12d5d314f14af223a5062bd0e52f1fe62feef4590f8b31a52d1f1c6b00", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.6927483410108835, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:14.020576+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "37cebe0f-42a0-4c48-9a83-3cc861d90445", "result_sha256": "3b3b407dc219414039f4cfc9ea314fce06f40f070b6cdf457e133ff6e187b190", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.9250608950387686, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:27.502384+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "64cb751c-8100-4094-84aa-80d5a71d180e", "result_sha256": "7662989f64d1d1a6513a5caa1bcf61df65e6387e8684f948932364a4db385cad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.924071489018388, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:08.884190+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4902fe4-ea14-4165-9e6b-877fd8632b57", "result_sha256": "3f2247f29558ec5040ac992d55af8a145118318d0bf590d925c874dfd14b5745", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.8790506310760975, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:05.558129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f974b7c5-e6e9-4fcb-b337-2b228e29170d", "result_sha256": "883f21cbcd770939b76fe54e070e42a86e919e165bb13d04c2664d2099f7c96b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.7283834799891338, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:08:43.613607+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "38a5a9f1-bb76-4ff4-a699-68608af8af49", "result_sha256": "670ce2d701f4e2b7801edbc9b7ac5d4dfbfc26ffd13d82a03f9d869515f64b22", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 2.1898191809887066, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:15.508636+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2056c8a5-901f-49cf-a4f5-33377ea9fafc", "result_sha256": "dfa98f4d4d99687e03a46b6785c8d863dee99cfa824b6948dc7141eaf2cc527b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.7612474620109424, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:29.080278+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "622809b8-e10d-46b6-9b17-83bf94a339e3", "result_sha256": "e285fc967091be8785963a7809948080e2ea6fd26f22c84188469722ce0d514e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.8545936199370772, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:43.531299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7e9c22a4-8775-4f17-b58a-f2838f48a4e9", "result_sha256": "060aa5c1e08cce4e1b8c83de50aac08d96efb8e1856b7f5da4106f57ad684705", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.96852619596757, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:21.091363+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2115b9da-4950-4d2b-8dd3-76d9955ed0b1", "result_sha256": "0c765c1aac8a11121b51dac4219c5cda02828536771da4b8376d81b39e723331", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.7599883610382676, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:44.877942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "71ffbf0f-2b3e-4f1a-90a4-8c254f2ac8df", "result_sha256": "b194efd04929adadca0d7f9cde47499dd02ee1511e8155fffdec58a82e60e001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005726, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2808, "key_errors": null, "latency_sec": 1.9099239059723914, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:41.236945+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "63a18950-5603-49ed-a8ee-eab838a71c2a", "result_sha256": "8bbd913fc95cebd0d8ec45cb13cc4dc1e19269d6c64956647c3e5ed90a17eae6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.6435424860101193, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:10:01.151396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "bc52bed5-67c8-40c5-8d79-2fad060e657c", "result_sha256": "d526b99e6c1726a9b415a2d598ff69f548dd26ff9b08d5cedcefa249bf2874ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 1.565758976037614, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:24.119232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9918650d-ba20-4bd3-a30a-367dddf72093", "result_sha256": "0615736530bb038334ab0f8ac2ea4a20f620cf367086218b32d27d2b3c0d8093", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.077913255081512, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:21.151905+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d09a730-8b45-4849-a21c-5c8bb581eff4", "result_sha256": "e6b72deb999e0295d0ebedac3e3f80f89202e580f1069847c70d474f6cc89da4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 1.9920707419514656, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:38.225816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d152738-a4f1-4ed8-bca1-658f67865944", "result_sha256": "b5133dcd8f9e6ffb72092a65a749dc57b5471f556b19266b4919aa71f790067c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009278, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 4.027472860063426, "model_name": "claude-sonnet-5", "output_tokens": 359, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:47.331999+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d2f4646d-f4ed-4adc-98bf-267550217eb3", "result_sha256": "9eae7fe77acd9fcef8aca99baf02c86b686f134a6de159e99cc08a725e8951db", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.098136610002257, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:08.638424+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d27c08b2-df66-49f2-bc93-077680d2dd95", "result_sha256": "483771da25d2df17fba02f16533ddd5b39a8ba9aa37fefcc9e2c728337b56f60", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007788, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.0621805479750037, "model_name": "claude-sonnet-5", "output_tokens": 210, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:43.871731+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3cb95ce6-b649-4ce9-bc36-ec833f5ae3d0", "result_sha256": "89ddb032f7e1f52dfe3a38afc36146cbae5b0968babd6627cb80daf6dc89cadb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008358, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.786212519975379, "model_name": "claude-sonnet-5", "output_tokens": 267, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:53.969419+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "308a6759-f4ce-4f06-9a67-93d5dc59e33f", "result_sha256": "3d9fb6a71c030b3d8b508cb65875f9752ce5882d0bb776438d5d36011da3bcb9", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005798, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.14046645804774, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:36.727083+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b41422bc-f6af-4b89-ba16-2b43df407d45", "result_sha256": "9ecc198c3cbf0eeacb2a642a81111113020cc0fd9639be03651d52425fdc843c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006848, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.273903591092676, "model_name": "claude-sonnet-5", "output_tokens": 116, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:41.391503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4aae08ab-8f60-4988-af40-9638874aa5fe", "result_sha256": "dc6135559a9d43aabd2d81c60ad9e7b0b5c8cf2aa968f9ba980452e8b6bdeff6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007298, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 2.8037703649606556, "model_name": "claude-sonnet-5", "output_tokens": 161, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:16.633366+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b65e6f7-91d1-435c-a156-c24d88226f25", "result_sha256": "fe9469509c2adab7dbd3c021eca61ed024602c374bddddcfd7ad7d846f221b8f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008068, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2844, "key_errors": null, "latency_sec": 3.3190621440298855, "model_name": "claude-sonnet-5", "output_tokens": 238, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:34.204824+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e869558e-cdc5-4be4-afb0-2e8fa47bbe24", "result_sha256": "c072fdeb0291412986218589ab79f8396ec34cb030336107e37c613c2deb29ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 3.5639592779334635, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:49.140200+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2819d12d-a8a8-4ca6-9e6d-9098fceffc49", "result_sha256": "0a6c1e9c210d76eed1144121868d24aed0eae3775d34eacafc2af148040e1681", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.496376744005829, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:01.080572+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "76897cc0-4b1a-455a-a74c-20a366a4329a", "result_sha256": "ea46a8804c2493057a24ed68bc8251d41f47072ced18afda0ca9611310089061", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.7562642869306728, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:21.300758+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "77db33f7-8d61-45eb-b1e5-3953964dfa14", "result_sha256": "78bb4fa56af67c9c55af58b3432ce7f0ff6f0e6edd4dddf9f23741e7c8a4a875", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6399901380063966, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:53.040276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "30403d02-6376-4c89-a150-4d8bdafe7379", "result_sha256": "ac499320838b246d623b03be7031a2f2babece19898ed6e469ec54c95818198e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6765949019463733, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:45.686238+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9e56789f-5b01-4e54-9dc3-2a9d55082cfe", "result_sha256": "651a158e3bba2a27f50ab09cf0de2fb1809cefa5bbfda7bf9e2356f8a943cf47", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.7073041759431362, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:19.393702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2e560c2a-ecfd-41df-87a4-7fd5ea679065", "result_sha256": "094661c5e4f300787a3fe3fbbc44f3960492299a7fe3ca626ca720e4727da425", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.8715016409987584, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:26.342976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9dbedd2-68fc-4b60-9d80-862e7923779f", "result_sha256": "e11807393e3487abcaab5f7dd553da0f071803bb7190d5adb9e3bf0b4004cb7e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6559726300183684, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:44.711823+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a1e4cb73-a041-4281-a98c-c4f597921633", "result_sha256": "605912c789683218871da0f0759d6803d5ef7a304977ec83240df044214430f9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6405322019709274, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:08.730551+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "176ace15-e243-4bb8-8840-870049bcaf2e", "result_sha256": "f3295dcb712d55d9d5b0ffd166b2a381060bc74f9fd29cb631c2a291b872c1cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.8977587610716, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:11.082862+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5f342bae-b918-48d2-ba4d-8cfd2086653e", "result_sha256": "c43f6ca93be2c38451894ee74a0ecaf998a381011e3481546d2e9ef980b68afc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.4364301969762892, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:32.263730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5a239266-766e-4b6b-928f-d565326c3663", "result_sha256": "732b26903e0d6e91117c0dd866361d4a0aea28533b7dc41364409306c983ab1c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.3847472199704498, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:54.856851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "58655d4f-164e-4baa-87d7-9fb799a3883b", "result_sha256": "6649f1225331c5b572805dec08b9533193410be5413672fdfd9d2e4039ea2c4f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.538970006047748, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:16.229623+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7f9694b7-12e8-4130-87a0-9b54321f8229", "result_sha256": "16639bc2f1dacff1c62b25bcf06b2b45d8346b9ce8dc2014b388665f8dc40077", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.8654500189004466, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:32.157474+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6c3705b3-603b-4c6a-96ff-de52c6a7a825", "result_sha256": "6e2d886fcd426eaf36e8b28728b0d82cb67b72d17905779c06e76e970f219b44", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6365681189345196, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:39.006785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d7273d04-11bc-4c37-839d-b45a60361256", "result_sha256": "938c8396799e1844e13c52f4497c4096724e4258443e449095fb69df02d6dfc3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6547739380039275, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:32.031426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d101460c-3f11-4fe4-81a3-388e3b615632", "result_sha256": "3e140f706330c17d94912bd79465b1b0b7a46e7e25ff5e9743233c525128eba7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.6420017270138487, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:11.067395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b729837-3cba-4b1c-aa93-0f2f3a295e00", "result_sha256": "a71d73e74be203b36e7597b19defb75d07a7fc1ea5626e49ce2a705477a4cd5c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 2.010983929038048, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:29.255982+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b3107fe-13bb-424e-a2d9-5aa846dc6980", "result_sha256": "cb112b188a30322cbfdb561f7a1525d60561b35ccf17b69f3cfa851c8ffed320", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.8352821399457753, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:53.124213+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a6b75850-8297-462f-9558-fb7c73303ef4", "result_sha256": "504c7725942624446918763fb7466a594fe1ddf704845c7030e150e64bbdb3ff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.3241011969512329, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:45.513871+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "74a60c46-4a7a-40b4-bd7d-6ca425b2f97c", "result_sha256": "3c755aad0e8f236d5f672e99821f93f85ff7dfa04de13e76db37cfcf860abe88", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.8184166129212826, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:27.668709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6dbe860b-4d38-42ac-9238-87b30b0c9cc4", "result_sha256": "f4ff3bbd56007e379e2daaf72fe5b7d516376e328a623789acb378fdeb53c26b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.738166572060436, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:04.499928+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "322a3276-3ce9-417d-be03-206b24e7a60f", "result_sha256": "8b11b50873d3602cd42edd6f329c4407000ebdc044a8f1541d5fafd7a858f434", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.60539149295073, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:37.087450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7037b751-a0e8-4577-b4f6-4603c33a15e1", "result_sha256": "8299c9d0e5f99fcd589d2bb007d6922d24fab58404c4a540b5db1d02fbbfd620", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006252, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2826, "key_errors": null, "latency_sec": 1.9759991340106353, "model_name": "claude-sonnet-5", "output_tokens": 60, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:05.964531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "89d0a135-8b25-4585-977d-f1a5742df906", "result_sha256": "3216fe7ac8493578b1890a81a342218de3f9c7121d03bed710125a9b4ed4339d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006348, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.027626106981188, "model_name": "claude-sonnet-5", "output_tokens": 67, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:21.114515+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "902d0c9d-118f-49a3-9eec-e3f7ea853a76", "result_sha256": "754b42d081d71295cf401477e733c51d16dbed2391c28d40d8baeed97de2853d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006478, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.3372013369807974, "model_name": "claude-sonnet-5", "output_tokens": 80, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:35.579547+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d282b695-3594-4f5c-bb5e-5f4dedcb7673", "result_sha256": "0995de784a343429076e60939840ff51c3045c64acd320de93eb3cc3d930722b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006388, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.388393223984167, "model_name": "claude-sonnet-5", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:28.366528+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f6f9603d-9db0-4c8b-ad38-7ed9df90f7c7", "result_sha256": "2c1010c348df588513ecfe2a8d7343560e7c94b318522d0417a7901b020b2dc9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006638, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.246818169951439, "model_name": "claude-sonnet-5", "output_tokens": 96, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:20.156066+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5136a5bc-14ea-4f63-9f2e-ae2bd19a1f12", "result_sha256": "bab22a664994dc4f4784b13f99f4cdb3fbe3bebdeb66fdbe59b2342166bae641", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006218, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.1516330160666257, "model_name": "claude-sonnet-5", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:03.126041+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "76b05706-34e7-4219-a5ce-7e0d68ed432d", "result_sha256": "aaecf1835f5056a13a2d9a2df3bb0990fb13623551165be735aaecaac40c1803", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006248, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 1.920984479947947, "model_name": "claude-sonnet-5", "output_tokens": 57, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:02.765948+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "765429e5-d611-4e11-ad60-bf68be0167bd", "result_sha256": "0ac7628ab7cfcafe14d13bcf6f35d8c43bb486664752ff3427fede5d95b77081", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006218, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 1.4037302699871361, "model_name": "claude-sonnet-5", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:33.569262+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "90c71a40-bd48-4c97-a1d8-aab447337775", "result_sha256": "1449f003fdff4e65bb02e1de55058cc50931a5830d46ef253cbb0fb2d24df4a3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006128, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 1.907714041066356, "model_name": "claude-sonnet-5", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:23.254031+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "87ec7479-8f79-4132-ae73-78d4deef586e", "result_sha256": "cfebdf0f740873adc07dc0fb273b948d0bae0b804d08c157c48361fe50fba313", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006418, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.1001968829659745, "model_name": "claude-sonnet-5", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:01.866298+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "05e2c8f4-46f2-43f1-b621-77a3e615462c", "result_sha256": "964dbdcf5b698dc6ccedd7f2f0f571be66cdb42518b621e9e15ee381e4f69669", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006698, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.471203427063301, "model_name": "claude-sonnet-5", "output_tokens": 102, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:28.709856+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "98586356-3609-4b23-a579-0ab3b937bfff", "result_sha256": "52e7e89e978e06310adc57b2bf1e2f7e4f728ded8e2dc620d429b3f5616fe8be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006728, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.381715181050822, "model_name": "claude-sonnet-5", "output_tokens": 105, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:15.207651+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "03778fdf-9ee7-4c87-8a35-b06b3d2ece84", "result_sha256": "2637a145049419a74c490b1e731445c9cc2bd3c90b2fb09e909d3a3b20bea5b9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006768, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.17608530796133, "model_name": "claude-sonnet-5", "output_tokens": 109, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:41.877101+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a8510117-86f5-4199-835d-ec5701cf7b73", "result_sha256": "c30c12567e97e05c469933dfa1e06b33dd7758f7c4e791260fc1cf469a5b9280", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006768, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.2529247290221974, "model_name": "claude-sonnet-5", "output_tokens": 109, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:06.894988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "897ed765-9a93-4ccf-9ceb-85ee3807be21", "result_sha256": "604808281c0bdd8a7a03fa575c8919f7d2a025a5dbb5a4fc6d850b03f3784718", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006838, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.145954924984835, "model_name": "claude-sonnet-5", "output_tokens": 116, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:09:32.824564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "dca62530-4a35-4de4-90e2-3f919bda2daf", "result_sha256": "0e8c0df029d0c03646f438f7b93dc9b78fb904ac5ae921d10cbae008d2c2909c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006538, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.4513864050386474, "model_name": "claude-sonnet-5", "output_tokens": 86, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:36.225073+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0e509b79-dd52-406d-8b0a-2069f1707154", "result_sha256": "fa8541e803bd3eb7a3771dfbb48f47330fcf2eda7856a467d7f0187b26eb8284", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006888, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.5750955160474405, "model_name": "claude-sonnet-5", "output_tokens": 121, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:53.229047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "eea36abf-d9bf-42d6-96d9-ef4dfdb6dd13", "result_sha256": "7d5e36ae23f0cc6f17ce1da5f92d36930edb97923861d5f000fbfbb003b51128", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006598, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.3516844959231094, "model_name": "claude-sonnet-5", "output_tokens": 92, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:21.259662+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9f0b39ee-b5ac-4e52-9709-6e1b8f95a7f1", "result_sha256": "8b8208ab65736f7a460dfd124a80f4dd35f45e7c361fc5fd13c8435cd5d87598", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006858, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.8583443220704794, "model_name": "claude-sonnet-5", "output_tokens": 118, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:31.189006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a9773ea8-1dc5-4fde-8c74-71a0605681fc", "result_sha256": "7df5da01e429064ba1d9a05c2884672bee57841d911d4fbe0317871c0affe883", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006578, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.1329773040488362, "model_name": "claude-sonnet-5", "output_tokens": 90, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:09:10.274628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "09f146a3-3db0-4172-9bd9-276eda9655c5", "result_sha256": "00b2edf4f67deabecf5d1d8b7fc8be761fdb1c9f89c795df8cf0a8505cc998cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006458, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.1829612720757723, "model_name": "claude-sonnet-5", "output_tokens": 78, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:11.223863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "bbc109bd-29da-4ec3-b487-389bb5a7e057", "result_sha256": "eed6356d617be07f0bbb5e65be6f60c31fb877df6028756bed644717c3d5bbe1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007168, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.8970667470712215, "model_name": "claude-sonnet-5", "output_tokens": 149, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:46.186625+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5832d5a2-1cd0-44bf-9425-56acf72dd8c5", "result_sha256": "38cbd91d57e131b7de87798492f80aaa643a3e94353840a8107e30a71c7fa16c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007058, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.854242269997485, "model_name": "claude-sonnet-5", "output_tokens": 138, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:58.219523+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0c4b825a-e894-4d7c-ba90-c43dd38b447a", "result_sha256": "eb15038a385846b4669d3ba1d4c09d18f6622890093d364edeebc0ef4f297d29", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007288, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 3.4899997679749504, "model_name": "claude-sonnet-5", "output_tokens": 161, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:27.686949+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "c6d1cb59-5579-48c5-b6fb-17e88354c25a", "result_sha256": "fd1980c3dc4ef7cb81603733f90d99a8b73f92b5e71884be8c0af475609232d8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006788, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2839, "key_errors": null, "latency_sec": 2.7119107090402395, "model_name": "claude-sonnet-5", "output_tokens": 111, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:47.624454+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e0c5230-cdd7-4e07-a782-4738c3fef22e", "result_sha256": "a6c3154e59ed15e866ace8d3b5f04c93f0c51ddf2c14385ae025375eeca76607", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007046, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.0402870930265635, "model_name": "claude-sonnet-5", "output_tokens": 131, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:57.409889+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "588cf478-c3fb-4074-9e68-58c662bafba1", "result_sha256": "796c700ab463b4b20a90fb3237e76b08315ad8724b68dca46bae1a83c24fd073", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006956, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.1850441429996863, "model_name": "claude-sonnet-5", "output_tokens": 122, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:18.994471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "90528f56-072b-4dc8-899d-aff7b0c3848d", "result_sha256": "85db66eb5a429f2dd1ca5605718be488dbf7231b8bc694475e5c04ff85441576", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006886, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.0162721569649875, "model_name": "claude-sonnet-5", "output_tokens": 115, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:11.817050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e7ee93f8-c5c5-4405-a134-41daaa774424", "result_sha256": "31e512ffc475f27fc01ec11e49b79c3da0d231092141c30e040d55c75a198bdf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009456, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 5.19592324597761, "model_name": "claude-sonnet-5", "output_tokens": 372, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:49.054107+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b89174b4-8cc6-4b5f-ab7b-d5f0e6bc5efc", "result_sha256": "41d17b3eb45cb98b8cb5cfc0bcf5e2ed37893dc56eec63c71f045f17d469746e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006876, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 2.6878300880780444, "model_name": "claude-sonnet-5", "output_tokens": 114, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:26.755259+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c2a6ece-b99f-4dc3-89bc-9221697d5ef6", "result_sha256": "2f9d2d122d94e93584504c9a5f441f06581515bada4e648baefd43784f248257", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006926, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.174259925959632, "model_name": "claude-sonnet-5", "output_tokens": 119, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:10:04.580142+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "df5b0fd9-0d14-41c3-b318-a0667556e8e5", "result_sha256": "345c0b77a25659596dfa1eca767bbc95aeebcd2955975425956986dde193e232", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011486, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 8.0484708850272, "model_name": "claude-sonnet-5", "output_tokens": 575, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:19.287734+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "dee6f1b4-f777-4562-a596-1406a69c38dd", "result_sha256": "ced3b52da0a77ea53a56ecb7d79f21620bb5f0ae5111df7387b3171351e840fb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006916, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 2.5784367159940302, "model_name": "claude-sonnet-5", "output_tokens": 118, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:43.264221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "09096646-df2c-4b7f-ba50-eb27161a39e3", "result_sha256": "581e8939942ff1e95bed385a6eb0f5111b42eefd012d841f68947284899f377b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006936, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 2.376766767934896, "model_name": "claude-sonnet-5", "output_tokens": 120, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:36.981866+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5768b4e-01e5-4f6f-9379-351df3f0811f", "result_sha256": "1431fe1d7d57bf12dec82c9f80ce3a04ce3fddacfe55272a6bce14ae9362db4d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006826, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 2.550000458024442, "model_name": "claude-sonnet-5", "output_tokens": 109, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:47.010020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "fd424cd4-9542-47c7-b7f7-1e690fa45276", "result_sha256": "df537b42e1c310a99e3062aed280bbb5236fa1b76ae45a3d1b48f29ddaef9f67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008126, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 3.2664555419469252, "model_name": "claude-sonnet-5", "output_tokens": 239, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:33.374710+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3b96004e-e1a0-4bf2-b4e3-04ebe03894ee", "result_sha256": "ddeb2fcea10b9b95a8e2b105e666871a0ad56a852205775bc12fc82ec2158d15", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.007026, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2868, "key_errors": null, "latency_sec": 2.754816284053959, "model_name": "claude-sonnet-5", "output_tokens": 129, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:59.880106+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f86f2c6-1483-488a-8618-ffdac5300c23", "result_sha256": "8c195293c18a2e66b30593675d4665b9d9539fa54939728814a163e79b9550b5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00635, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.4031200320459902, "model_name": "claude-sonnet-5", "output_tokens": 65, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:19.340228+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "457693a0-ef04-414f-aa69-5eb670c8fd56", "result_sha256": "96eb63c93f71bbc3cd99b6fdf62e78bdba2f59cc91dd9e959c352337e9c1bcb7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.706719771027565, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:08:56.024267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1181ef1a-6976-4935-8441-7efbb5d599e3", "result_sha256": "90ae22a269deaf0fdc4b81367dcaf28e227e65e0b96bb68cf1d7b9562101ec59", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00606, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.122168380068615, "model_name": "claude-sonnet-5", "output_tokens": 36, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:46.177855+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "70b0bb6b-70cd-4238-9d58-648837229856", "result_sha256": "0268a82a1c41c7a6855153d7625bd3195dfda83427d3731123be78f9e9f0f934", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00618, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.8572990730172023, "model_name": "claude-sonnet-5", "output_tokens": 48, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:09:57.576340+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4e024308-9246-4384-8f03-629c828538b7", "result_sha256": "da5465c39c79a4858411eeea75a2830fdbf591c88bf2fea74be4201ec8b9a5b7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.95495464396663, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:48.343616+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6944f9c0-14bc-4041-82d2-c4adff358b3f", "result_sha256": "d4a3988ef4d73d965ddfaf5513973ae36c4351758d46b9e6395f86bf356c0b7a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.6920058819232509, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:37.348761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "91ca7e97-63ce-4cd9-83c7-0c04c67665d4", "result_sha256": "1d9d86c9f395e99fe218b4f1a78df272017164ad03d37f440ad7c03ade6dcc74", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00612, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.406928612035699, "model_name": "claude-sonnet-5", "output_tokens": 42, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:13.363232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8cb2625-5cde-4094-96bf-4778bfe2ad31", "result_sha256": "50fac827f107f0239324c8049f60ba2483f1981c17720b8d7bd3b2a221093403", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9074537679553032, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:14.263425+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "11324dbb-d328-4d34-9a2b-e86ad5a4bb9a", "result_sha256": "38f72fb203f49780afc1af3b55333aae47e88025f3b6379a0263f2f09e7b4fbc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.6353563100565225, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:19.811651+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7202e0b8-c313-490d-a65f-dfbcc22a1a68", "result_sha256": "c5cfba4a63b2b06be64ad8dee688aeacf1889960e96d1cea5e508c88b1c3e91a", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.7050390880322084, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:53.226779+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5eba811-e7dc-491f-84d2-d0ef78b6bcd2", "result_sha256": "e47f3f359bea6bf7cac36a9142778ff94ad294a3c4545ed6f4f5b91d32d51f6c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00604, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.042343668988906, "model_name": "claude-sonnet-5", "output_tokens": 34, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:01.294678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "7c269bdb-8581-4236-b6b7-305ac53c76bb", "result_sha256": "c2cad2981c928c317950c2cd0edca97762fc16b73e5c796ae08438927dec2722", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 2.464632790070027, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T12:10:02.503320+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "bfd00c89-6317-4c82-8e4f-65ddaa799dc4", "result_sha256": "b725dfe06dff11334bd42e8e1d466e6c0aa7e43f2843a2318cbe6b662b754614", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9366119540063664, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:05.364654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "da32ba9d-ea0c-41f9-bbed-57944a9727e1", "result_sha256": "f6466a504d143b9c07cbe09de8e27201b9c9ad759b1eac1026761ad5bbf539ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.678826026036404, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:06.520148+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "10119a25-7b5e-4675-8a72-7687fe9049fb", "result_sha256": "ccc3133ec2267038b2f83794e5ff9a3d0c1c3228ea9537ba4a53cfff3e326db4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9496594640659168, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:37.283887+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "72b27a46-2bd0-45b3-ab40-8fdb8bb3ead7", "result_sha256": "598a816ecbf5e74c8e848439efecfe341c93a927ebcbadecf68bac82691a5916", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2850, "key_errors": null, "latency_sec": 1.9570697660092264, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:57.413717+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "cbf24630-86a8-4cfa-823e-4aeaecd2699e", "result_sha256": "3c29328516d5c23c06e033f15ed4047e7e855c253205a69e1bb48273b8d8a183", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.7822195639600977, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:49.817121+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "63a2af44-75a4-433a-b6bf-cda5cb7bd198", "result_sha256": "bff8c6fe5b264d0401a5d5390f0886fe289a595f0d3b761c281351b0a9d924cc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.9383013639599085, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:29.726013+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c8adff1-942c-405a-8570-4f23f3a3fec1", "result_sha256": "c90737276a98d2f0ed71637c148c843a16d4f070b1199a341a2917600c635418", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.8292600100394338, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:29.776518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9117cb85-22a6-4619-89c1-7f37f30d9c06", "result_sha256": "44f177fc55288bea4aaea01fce26c57303e97d05dc400dbd5b073cd2f30c5414", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.8992383549921215, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:25.436106+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a32f575-bbb2-4d6c-96b0-d0a3989e7a64", "result_sha256": "e8c1801183436814ac203d601ab55e3bc82ac65e46b5b4b18c180d8b6cceb6f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.0320713290711865, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T12:08:46.487840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9eb60085-6ad4-403e-b6ba-99dc7b298796", "result_sha256": "07bcd4c965a2fbf2eb773182261d3c19fe3af201ecb482859ee72c1ab64e24fb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.780485172988847, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:30.910026+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c3cc2c2-bbb9-4f18-a88b-682c4af6e135", "result_sha256": "a298c7118b5fdcd04cf40d8bad21ee02dd37c254218ce93cbe03274e9320c70e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.8905104349832982, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:25.614237+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2462e76b-038e-420b-9950-26d718b88d6c", "result_sha256": "d6694de6b6915acce21d8d908e866320386cbb1ef45fef1865cfef0a33ccbafe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.042702225036919, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:47.130523+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "226d4975-74fa-4e94-8d78-e081c129a1e5", "result_sha256": "75d66476bb4c6ae1a83724acc1ab9b8af2e4bd07aa3d1b2c9bcd41bf75702dda", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.9959734079893678, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:59.221900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "11a0851d-d6bd-40ee-b56c-91062d90539e", "result_sha256": "e7a19042d58c7d59d9a4575203970ea5186bd942454efc1d6f338e1ff7ed092b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.996524781920016, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.877074+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "02be0905-034e-4341-8081-4073661016e0", "result_sha256": "c2dbee0b3f55d88bc25496a912cb4aaf15da98e2bbd4ee7afbead19b16a9b2f8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 1.6579577049706131, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:45.344487+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "6df8723c-cd20-4e4e-94a9-3f8635f21e88", "result_sha256": "9a078713bfab7d1b8011d2d2c0bfcf3f314bd0e471c3613ee78b1d5b0b9be9aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005666, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2778, "key_errors": null, "latency_sec": 2.227374981972389, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:57.236190+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e562c17d-98d2-4661-9945-30eb2e323c51", "result_sha256": "4683737ceb2d08a2fb28bee6b5bb1ad777d50cfc235c99bbd112ab7ad6baa1dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.4395100580295548, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:46.747650+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "64df6964-ba69-49a7-af62-eefeec7a7464", "result_sha256": "ec16b4e3916fa31c5a08a697f4ac28916b429ba5451c3f56d236e9edb7df5719", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.6969227449735627, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:12.289844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8add293c-2eee-4849-9274-85e790490510", "result_sha256": "78b72ff674e539a5071c39b0bfd9f7b4c4f648ae4d1f7309300da962eb7649ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.2586666180286556, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:00.629458+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "67a0c9cf-5dd2-45d0-a04c-88ac2fb39ed9", "result_sha256": "40aeba9589e24d1e013b57a8f716c34044127a4a949456d584c150f24d53769c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7167392029659823, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:07.437927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b2f8d4f0-bc84-4dc6-bdc0-4f1511f4c20b", "result_sha256": "9fa65f73f7a0895acade7162b21261ba71d4ce65be8170440c3a2f36e9e2b36a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.6545872570713982, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:19.279698+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "924bb918-71c7-424c-8333-1180ddc8d216", "result_sha256": "38affa70d3393be0816172d252edb81ebcff0ab2c85390a75a378772cccb988d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7675636460771784, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:11.775772+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a1a7e5f-8735-4df7-8f9d-5895091283a3", "result_sha256": "45e82d3a24ebbffe43cb88815dd496fed3de53ee15d3efefaa290be5ddbad0cc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.761712172999978, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:08:53.693343+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "39d059c2-1ea3-41c4-a1ff-13ec84109f6c", "result_sha256": "67bbc4dccbeb1e3429e64877d75e021a785c5440530811a449971bf5dc97e526", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.446990331984125, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:49.528710+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "db85d6fc-6c94-43f7-816b-d948e4074179", "result_sha256": "3777c267fd32d4e2e9dfc71c07c8696ddfb988a3baa98eb8ef458f5cdd07e651", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.5140671629924327, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:23.126237+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "bb007ec2-70cc-47d8-85c7-e619b6d34edd", "result_sha256": "8e7fe35fc0d9957a742b66702910419fff32bb8b32ee441fe7a3262455c858c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.5874728410271928, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:34.168684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea697977-f094-491f-815e-4195c83af95f", "result_sha256": "2da86a0790e2f735ad2ba34e748a8bde7f8597e6e5d4cf66b23170e35536efed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9576499729882926, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:50.288458+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "671cbec2-efbf-40b7-8470-635b99ddf5b2", "result_sha256": "f16e9faef20165b38f205d2a5b971f20925426a5163c0f6b3ab276fc7b97013c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.6208739780122414, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:35.410610+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "df4b8e02-202a-4720-af05-240c3d35d309", "result_sha256": "60e8027f99248b33416e69f049e09d17155cc3ef3dbba49f6ea540746c5b3c5a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.416442007990554, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:53.734551+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "76fc0153-93e3-4cb3-a380-b6ec90b3d30d", "result_sha256": "5d62b83c49d2cb3b56281ee664ac8cd6823d434d24933201194d306a4f45cf10", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.9379101989325136, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:35.698082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e62cf3d1-3c0b-469f-bde8-637477a36e50", "result_sha256": "47b21d647468b71cdf91f9efcec903dee7fcb3ab4ae9d7dc425af6eae6340773", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7125317560276017, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:08:37.652188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e3dbe42b-ee2e-4c05-88b3-f36485e6bdeb", "result_sha256": "3ea5eba0976d7f266bb47df9b49d553c7ba98f20f9cb287b03d564eddceedfa1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.863409185083583, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:16.725062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "927f63f4-3fd6-496d-9a29-2ceb4265b024", "result_sha256": "a7092492fae512d51a6ee188151e22c49f0a842d42ec7cb4c8fbf75cb4f7ef8d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7343878019601107, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:57.334289+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e1758252-231e-4934-aa84-3e9bda34e533", "result_sha256": "de3aa5a5b34788297be6e2ebacfcbb403159ef156cfc6bdef158d08ec874492f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.5548049189383164, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:41.822912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "16aa0b8d-1c0a-4f4c-ae4a-99b28618b3a5", "result_sha256": "6cc11b82efd77349e9c8d5a0a8cb7a983e6aa3337439f3c8b280dd7666e13b73", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.685533793992363, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:54.315286+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e766eaf-a25c-4799-9354-a1945289af52", "result_sha256": "c6af8fab8dfc9a3626ef90d02422fabfa49f4b534e3931be5f519ad6bb4a0798", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 2.2851100439438596, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:05.745627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "aac07c5a-9ad1-4077-b6c0-c1b6b7b1db94", "result_sha256": "ca2cd719e9c8efbcd608e0b83ce573a2e5270a1025e0910d8252996312abe1a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.3417717290576547, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:20.004541+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce757ecc-91f7-4795-b8e7-2c8869764cc2", "result_sha256": "cc4210141a388beef0c38cf5c864ec1bb48f3c72c3da51dc2420791bc9bcde3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7976116710342467, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:35.614775+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d605379b-58ea-4fd1-a1b5-e530db6bed0c", "result_sha256": "5b5b94e481a44f92e21996294a7d371a0bf09bc9c92fe9632b9255d835392fb3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.7225757649866864, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:24.938181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "64ea3531-1160-4ef4-ab54-b61af5e14a2c", "result_sha256": "242ce2731e46ad8308a353218c23e55191234b4ce43953750ff3351166e24b94", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00575, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2820, "key_errors": null, "latency_sec": 1.608392923953943, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:23.048954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1dd13dc8-5c8e-4572-a77e-20ca64343f7d", "result_sha256": "f62738bada997203c25e3495651265e73455d0c14ebbc9e36797335965138f6a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.2724044480128214, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:35.428843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "09257555-e27f-4962-a00f-0bb3c9a18ab7", "result_sha256": "fa468755adf7156ab13bd9a8a93acaed1b13bcf1bfc131e94777d3956bf90a0d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.8035372260492295, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:40.582801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "74f77e63-eec7-4cd3-87be-b28df6ab4c12", "result_sha256": "8c58059b384b7ba7562cf2c4912715613e9d748cdd1477460e4c37bb7fb5611f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.6547564569627866, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:50.151065+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "850d8254-f8dc-4a64-b360-f081fbb7eb82", "result_sha256": "e1d30b783807ca00120deceff522f6a0126ede2b218be8d3fc49df13011775d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.182488483027555, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:08:36.912573+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "78a55406-1370-4267-8d68-1be76014dba4", "result_sha256": "d68f84d25677beba3bc234b633e354bb63f5b843076be74af6b68fcb7efed148", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.428717163973488, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:11.887615+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e5fa4191-afc4-43a8-81a6-7f1645d37c7c", "result_sha256": "e3a440f27a1b76d848656fdeafe2d4b3d6e1464408135e2e8a53383bd2ba8e7b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.009283086983487, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:58.862846+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "72d4c8b5-99fd-490d-a827-4776e5392a76", "result_sha256": "c997600f3451adabc828b4a753e39506c7baa74e3fa0b635f7c198425f0cd062", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.126288832980208, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:00.400157+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f1b4ea3b-1ef2-4cf6-84cc-417e9806aded", "result_sha256": "cec4c3b3cff7c444a296fa65e20b6ac350d2dd1b1ba68b45316831bac232e009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.8590241610072553, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:16.176802+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "024aec65-084d-4b06-9632-96274683be80", "result_sha256": "258af5c671a007a770aa4f97bf1d55a9d849af911ecb89af9dd6bf7f0187d196", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0061, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.1329939799616113, "model_name": "claude-sonnet-5", "output_tokens": 50, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:57.719702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e5cd387c-37d8-46cb-8d3f-69a676da9146", "result_sha256": "415f535e570465046526263aba432e4391740abcd71eeba3193b65dcd49c6555", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.070941030047834, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:12.939088+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e1cc8ed-eb6e-4990-b222-4b2ee5cf3872", "result_sha256": "5693c5065eb9c618810838d1d768c7f2fb6696b905d6b4448664c56f1d3f51eb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00611, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 1.9961060180794448, "model_name": "claude-sonnet-5", "output_tokens": 51, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:42.587685+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "8af62e84-e868-4dfb-a3c8-e6d17802c6db", "result_sha256": "e9e784ee65075659fd4557821a905b7e799771369a4135062bb5dec983537780", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00613, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2800, "key_errors": null, "latency_sec": 2.0867390809580684, "model_name": "claude-sonnet-5", "output_tokens": 53, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:14.821234+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a44fd7b-043f-424a-8500-b6700838faf9", "result_sha256": "7d1daf39d2d42e72502a9a07b41c09f22f7c61e76108331b605c96a6166a20ea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006218, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.1986675900407135, "model_name": "claude-sonnet-5", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.608931+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "47355630-6a9d-4ab1-9de7-7d7e5f66a67f", "result_sha256": "a31dbcd722c8c1d780f5060be73abd0ad99c1690a30ee333c450cae86815fef5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005738, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 1.7593899499624968, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:31.968160+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b832b4a0-5fe2-48b3-919a-d79c758d244e", "result_sha256": "f7065ddba5c29e2098783dbb23cb0eb3e416f67410f2a4eb145a42d1bdc9decf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006188, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.053793561994098, "model_name": "claude-sonnet-5", "output_tokens": 56, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:57.946608+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "e42c714b-f9cc-438b-bfff-43e5e7b322e9", "result_sha256": "39e908977ce09d110cfb7c52634c1c2df78fa94a6bebc2f36375f0350beb3511", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006698, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 3.395356275024824, "model_name": "claude-sonnet-5", "output_tokens": 107, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:35.189977+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0e32042f-eeeb-4fae-84f6-cd65366c7324", "result_sha256": "7cbc949005385bcce20831d490d5b35fc7c15f33a6cd72828b062c4076de8bca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006348, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.8254505830118433, "model_name": "claude-sonnet-5", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T12:09:43.019307+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "80db896b-c2d2-406c-9502-6f93c2170291", "result_sha256": "40503a3bc45829e5d6da08e934bde47da7e5f3ac54be1749dd1d13d167ff6076", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006468, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.164799113990739, "model_name": "claude-sonnet-5", "output_tokens": 84, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:39.457109+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "778967a1-0e5c-4eb9-909a-1ad3ebc7cb9c", "result_sha256": "3427f05eb4e6da6c572889dbb90a263e44ed5b39ce2396958d835028252cee05", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006478, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.3738485400099307, "model_name": "claude-sonnet-5", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:52.265538+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9a20640-a88f-4974-b7a3-c89210522692", "result_sha256": "e79b1240b84555a8c698adce3162e12ca5370125dac8444dae5c81acce7c58ea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006518, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.399390753940679, "model_name": "claude-sonnet-5", "output_tokens": 89, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:35.309045+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f86aeee6-88b8-4533-b2bb-5f5fd9c2c82a", "result_sha256": "7e7a0bd7036381d52f9bec0d996423d369fde88d9983a9ecea8f19acfa56c657", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006538, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.4679402239853516, "model_name": "claude-sonnet-5", "output_tokens": 91, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T12:09:03.244947+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "2ac01a15-87ef-409b-bc33-128c14037420", "result_sha256": "be76acb2e2d0591779103d421792b5a0ea25b005b2f2052243b56dfd4f871653", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006408, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.375635551987216, "model_name": "claude-sonnet-5", "output_tokens": 78, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:54.580323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "68a385d4-2d7d-41dd-aee0-3d02e64bd88c", "result_sha256": "b72d44e93e6910033a451b7c2fbcbb1244bf5172b787b424cf9574e1df9bbdeb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006708, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 2.7715131839504465, "model_name": "claude-sonnet-5", "output_tokens": 108, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:42.143966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed6b9c1d-9a26-431b-9c21-ee5c28b971b9", "result_sha256": "f267df741a0fd83451562a6526f62cf32587ed2c3f68d8ab68993b99a432f1ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007168, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2814, "key_errors": null, "latency_sec": 3.30020812607836, "model_name": "claude-sonnet-5", "output_tokens": 154, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T12:09:47.812319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0ad403d-2a94-4e0c-a05e-b945fd56673c", "result_sha256": "8aba202862279922b9f42047dcc2bc78bf1bcf3d90c121d7de2e6a5bf6b54b67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.9543747609714046, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:05.262299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "112226d7-2619-4cbf-bea9-2a7b7e43781e", "result_sha256": "4afe3cce5c9670853057c92d2f1d422b0c097d3523c7afdf691792253b38b259", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.2775221570627764, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:31.211459+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "32577283-f446-487e-8c78-5c26031cf4d4", "result_sha256": "c2f740ba5bf7f07942d6abce2644f579bf79e10efd87189bb835fb801ffd7b27", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.7495486439438537, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:49.348588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e16f3d1-a7b0-47db-ad0b-55168604e119", "result_sha256": "5f8d33e550239e73f96bf9a64f172de3850c971eb4d7007af58705616708bbfe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.738939507980831, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T12:09:26.252934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "5ea3cba6-5456-411c-af7c-a28d126101a8", "result_sha256": "8f43a5d451ef29169634a1864ef4c47bf6cf55308a7b1b8f7badd583ca5ff2f2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.6666506249457598, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:02.209408+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "f487d2c8-22a5-4d6f-b6eb-13380dcafb05", "result_sha256": "3c5515406693fc24e8b4a7e1852a533ad9dc1076c29ee6851003195eb2a52447", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.7264767379965633, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:49.787147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "0d215415-4bb2-435c-a306-cf94b3fe5264", "result_sha256": "89c3d7f7c60c3deca9fb43924ee325e065ea0e75ca849f03c90f3744836212a8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.6203896190272644, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:17.469473+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4ca5e82e-c5ef-4896-a7a8-dda368d767c7", "result_sha256": "ee81bf316c8dbbc98580cd3ac3a1671fe5cb4d9050aec3f69bd870badfdc8119", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.7282609860412776, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:04.928427+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "b1292de1-2424-4570-b2e5-04eb46dfe071", "result_sha256": "6df19298fac597151f511c4f26e7869f23b208d6bc1af9ed7bf2522acc0207ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.8390053869225085, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "anthropic", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:48.604774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "d32c5f72-1afa-4a60-8d41-ab9fa45f443f", "result_sha256": "04f572b6149c563a2c342eec717fed9a6c84eebb6b6fb8639307961c68fba9ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.8325763709144667, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T12:08:48.530097+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "eaeeb1eb-de9e-490b-b17a-2c08ba0504a2", "result_sha256": "25d9e47870635e55d2306daebd93e5de3db019ff667a1c4e26935e4053d2bedc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.8077237290563062, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:53.751850+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "claude-sonnet-5", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b609717-96fe-4f92-b901-22b135e24df5", "result_sha256": "b823a750c3ff0c32a7aa802ed1a376d5034f6e2d347addb8f74d20e8b2595f47", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2783, "key_errors": null, "latency_sec": 1.8659404340432957, "model_name": "claude-sonnet-5", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:41.689477+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac1ac44a-9f74-49c7-a6ec-ed9e85f08c02", "result_sha256": "61f878d79eb9fedfaad6fa4a81d09418390043ba54a8c34b55dcb097e9c5fd12", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005708, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.968669519992545, "model_name": "gemini-3.1-pro-preview", "output_tokens": 285, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:52.610121+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "21c4b93d-ea3f-41f4-8a66-05e9e2eeed09", "result_sha256": "264baf796a170af1afe045b8be1b9c7063a200824d9a0339c52d2ad8b67cd291", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00362, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 3.7591694039292634, "model_name": "gemini-3.1-pro-preview", "output_tokens": 111, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:17.841235+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4df6320a-e4ae-4c3b-889a-c6adfe52d0dc", "result_sha256": "ae4d769710c89fa66fd917595575a6f691e771ef2249bf63997ff256c80db4d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004868, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.171445143059827, "model_name": "gemini-3.1-pro-preview", "output_tokens": 215, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:12.073785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "afc48e2a-016f-4156-8f8b-9bdfe3678a0a", "result_sha256": "04d5bc810a32fa9decb93fa5938d3ac60443cff4581c30b6f1088ab0d2d7e635", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005108, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 3.9145725070266053, "model_name": "gemini-3.1-pro-preview", "output_tokens": 235, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:15.656844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd3f7071-8f33-4013-8bc1-48883c349a16", "result_sha256": "4171803f76ee92508ed37830ab2333e778f9f48636c00e8ff4e8cac2c1ca83df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004136, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.386484388960525, "model_name": "gemini-3.1-pro-preview", "output_tokens": 154, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:42.624766+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc6a07ea-2d0a-4cb2-90e2-f155fa43f807", "result_sha256": "975c8fc7525a2807d03c3920f6b1be715bce5a5e3472e142d0dbf631fb8670b5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003932, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 2.7748812650097534, "model_name": "gemini-3.1-pro-preview", "output_tokens": 137, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:55.794395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4bd5e5e2-eef1-4161-85f4-ca94b10c62e4", "result_sha256": "ebe72341e126a8742f2e56c598eadb97874ccb747c890b68307a926d982ef2cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005924, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.29929928900674, "model_name": "gemini-3.1-pro-preview", "output_tokens": 303, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:42.522952+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "fad68ca7-bafd-490a-ac71-b7bb0c1dcb65", "result_sha256": "3929f43de53e0e49dd7f3329ca0a2193b84681571f8f17c793b00ef075043ae5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00554, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.0387306320481, "model_name": "gemini-3.1-pro-preview", "output_tokens": 271, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:01.330642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a88fe48-83f1-486d-aa58-2d404e9608a4", "result_sha256": "2f1b34bcde0e301c2e7290378192b156dd6faef3ae43741fea04df7d695eeff2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004688, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 3.6450624089920893, "model_name": "gemini-3.1-pro-preview", "output_tokens": 200, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:32.643272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4186f556-4904-4b11-bd15-8b64cefe21fa", "result_sha256": "38d6a01bd9f8e086282502042a1556c5a244da393269ccd46bc12b301aa75a94", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006044, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.510025970987044, "model_name": "gemini-3.1-pro-preview", "output_tokens": 313, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:55.555151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e2dd5f0-2f36-4f2a-bcdc-9cd7a459d948", "result_sha256": "fa5041c2a5e8a062bb9208a74dcc1e3c10db28b6d92528312ca6c282cf3746d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003812, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 3.312720200046897, "model_name": "gemini-3.1-pro-preview", "output_tokens": 127, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:32.534703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "48f87ad2-1f05-4e4a-9343-8f980baff7f3", "result_sha256": "c44825a46664d70cd35ae93b2eac9b90144a3cdddfa8199ce213bf37059d0d22", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00566, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 4.16043438797351, "model_name": "gemini-3.1-pro-preview", "output_tokens": 281, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:54.411822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "8222c5cc-22cb-4ed7-8155-6a04f42622fc", "result_sha256": "dee48a2b9175fcfca3d983d8a409afb1e08db54fa0caaece7b2f6e3572af4b44", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00758, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 5.824380330042914, "model_name": "gemini-3.1-pro-preview", "output_tokens": 430, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:00.559900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1ef958e-cb04-4440-8fff-b0df6b4cc196", "result_sha256": "0aaf7a5411b6cd50701a05ab827542c16ab85bb37259107faa7edbf238dbe466", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004184, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.0085124240722507, "model_name": "gemini-3.1-pro-preview", "output_tokens": 147, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:02.840598+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "57e56392-8dd3-44c8-9a05-4a13e0384342", "result_sha256": "08a2254a3c5e616799495674238a43b1dc2878e99b389a1f0b41d74c146ee43d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00506, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.622840900090523, "model_name": "gemini-3.1-pro-preview", "output_tokens": 220, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:03.152701+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a25dcb69-2e97-41f3-9747-70e4f56abf1d", "result_sha256": "307402975667694cabd7488b09eab64009a89d2a25649c675a1f3b1d23ca159b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005756, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.189881289028563, "model_name": "gemini-3.1-pro-preview", "output_tokens": 278, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:08.875944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "540ed51d-7ae7-4e60-8d98-325b3c64f56b", "result_sha256": "496857b595525ed72fd8d4babad9bdfad983f7520fe12c904f7334a8dbbae91b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006932, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 5.061335948063061, "model_name": "gemini-3.1-pro-preview", "output_tokens": 376, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:15.145134+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "0ab7769b-4146-4c7e-a10f-316a913b1e7b", "result_sha256": "266d199f0f97832ff2b6c8520032cfe2a8c44b18bf4a5f5bdf6e66a038286b38", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004652, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.4063302790746093, "model_name": "gemini-3.1-pro-preview", "output_tokens": 186, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:22:13.792226+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "5dfdc7b1-fa21-4777-ab0b-aae05bbf5db4", "result_sha256": "e7f9bb680088a2807c3e8f2e5367391db357555e125d6519ae9e274f137ccd90", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 5.466734130983241, "model_name": "gemini-3.1-pro-preview", "output_tokens": 415, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:18:47.544838+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b18b876-cd1f-4f7c-8d74-6b7d49431938", "result_sha256": "b4aa6b249c5fcd26eea2ba5561a92e6c6596b7c3346420557554da5018f4ed56", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004796, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.88671302201692, "model_name": "gemini-3.1-pro-preview", "output_tokens": 198, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:45.349394+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b499a767-1e34-434e-b5dd-b6351f239df3", "result_sha256": "82fddf659eb4a94581578ccbb76fc2985f66d783cdc2d8dcf42897844a36feb3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005048, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.8776702269678935, "model_name": "gemini-3.1-pro-preview", "output_tokens": 219, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:29.611534+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "67a582bc-b23a-414f-a892-1196660e9c11", "result_sha256": "9f3885a501979ccae469ecfc084fa9d566e8376caba82fdc703499264ebdc526", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006116, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.520851625013165, "model_name": "gemini-3.1-pro-preview", "output_tokens": 308, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:25.115271+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "397de3ee-bddc-4e1c-bc93-31610b37c60e", "result_sha256": "4c911c27ef1751dd68e04f9c4a9a82d7c48cf484ff68be02ddaa2b5f66c61cbe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00488, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 3.702692796010524, "model_name": "gemini-3.1-pro-preview", "output_tokens": 205, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:09.939296+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "bbca095d-7a99-4236-a939-06e6775b2eca", "result_sha256": "0d4081fc1f93f1a7e87706e5e8da8e502caf17e82e4091fbb7dd9d030c21e2cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005456, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.76258913311176, "model_name": "gemini-3.1-pro-preview", "output_tokens": 253, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:36.632674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7eace9c9-b330-4ca6-9d78-3c71575e4b0f", "result_sha256": "2035560b7e205867e837b8e93f9faa34308788a4d84d8a45e2ade91328c39ed9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005756, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.175443466985598, "model_name": "gemini-3.1-pro-preview", "output_tokens": 278, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:53.053637+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "831b5d6e-b8dd-423b-8aa8-4dfef1bb4b0c", "result_sha256": "f2a8cd3b113d17d4e70a286c288bb9f098b47f907ba8c1dbc10f8fd694988078", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004448, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 5.4734241019468755, "model_name": "gemini-3.1-pro-preview", "output_tokens": 169, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:06.626825+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf48b667-8d12-403f-9508-e55ef2614684", "result_sha256": "bb8670db7254960ffdad18966e147594c152c7e2847b5958cb97f4bb639cd6cd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005408, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.104723145952448, "model_name": "gemini-3.1-pro-preview", "output_tokens": 249, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:15:00.344383+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "16438409-0beb-4a77-aaac-297ff2361fb9", "result_sha256": "1c93f632d23fc5e3dee57689d66e66753828e0295784596d08ed6664f0188636", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004316, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 4.06521975598298, "model_name": "gemini-3.1-pro-preview", "output_tokens": 158, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:19.692762+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e1c04be-1eac-4a42-9ced-b39d5c2c6b34", "result_sha256": "8f407ccb8bc7907c6ab037c2421dc7a11fe6e6114dadbc8b2df9e72d19b232b1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003166, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.9893517460441217, "model_name": "gemini-3.1-pro-preview", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:41.668343+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c96183e6-5dd5-449b-9c68-8f55f2480f4c", "result_sha256": "1b7cc3540df6f075110bad5efcd62e7a52d297be6aa16b9a590024d1ff810fe7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005974, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.150075888959691, "model_name": "gemini-3.1-pro-preview", "output_tokens": 312, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:13.727232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "086ff004-ba40-4f5a-8a61-68f302d534e5", "result_sha256": "a2e6c1a1e9157fdaeaa434d69d54d696ff0ddc617ba5daba1e59724b84cbc0c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005842, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.079863735008985, "model_name": "gemini-3.1-pro-preview", "output_tokens": 301, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:32.387438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "e8be17fe-b0f5-4a43-9dba-b755fea92d2b", "result_sha256": "8b20ac6dd2675eb8802cd08d62628fac3b89180b16fb37e3192bff69c8283380", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003874, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.865652263048105, "model_name": "gemini-3.1-pro-preview", "output_tokens": 137, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:03.673834+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9056edc-ff41-4f64-afbc-f8fa6d3750b6", "result_sha256": "8d29dec85e840ed7ee09b1466b94657e4924f0aec8e84961879b18c6f32e093e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003754, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 3.6524202859727666, "model_name": "gemini-3.1-pro-preview", "output_tokens": 127, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:35.311368+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "662ade8e-0b5a-485d-9aea-e9efb67c231a", "result_sha256": "c7ff42906eda171f45ce3d2de63625beeaf8298f0a2ac088e5128458cc84fee7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003502, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.838105597998947, "model_name": "gemini-3.1-pro-preview", "output_tokens": 106, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:01.741249+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5e7426d-c08d-4851-ab61-86648e84646f", "result_sha256": "6ff4da402fc3ddb8cc8d3718c73b4184e22e934ecd21a50c80e6150bbf865216", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004906, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.542901128064841, "model_name": "gemini-3.1-pro-preview", "output_tokens": 223, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:45.016158+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "cb1a30d4-ef8d-4186-b6af-38a4aeacee95", "result_sha256": "239e330a3a5a921e297520e8771f8089c00001ea5a7bb84032417f26e2d41950", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004486, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 3.5737783110234886, "model_name": "gemini-3.1-pro-preview", "output_tokens": 188, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:19.277555+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a8177e7-ebbd-45dc-9bb7-5b8b930cf3fd", "result_sha256": "7645cb8d0d488196024c892dc34abdee54a5453ea8cf6205b28a7d7e4fe9477c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008594, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.614269832964055, "model_name": "gemini-3.1-pro-preview", "output_tokens": 514, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:50.417526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5b3aec0-3da0-4598-9ff2-50ac4fa7b2cc", "result_sha256": "84641a36dc868a78da9d5b3990bd4147b6acb8134a6d00bfbf0705be5ca4da0d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012818, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.414530080044642, "model_name": "gemini-3.1-pro-preview", "output_tokens": 866, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "google", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:26:39.567202+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "e245727a-9c3e-49a7-86ee-fac49ce4f1c8", "result_sha256": "fc1bf07c81a8edc52595d135bee0695d4f0ea0ce53841023e2282f98bd1f23ad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.013166, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.868612103979103, "model_name": "gemini-3.1-pro-preview", "output_tokens": 895, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:03.906407+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "33be57ff-f5d8-47e9-8c87-f5fb8d9280f5", "result_sha256": "00ed9ccc9edd3cbd796a91459bf37e1dc49c6a37812e7dea9774e00ad965d09f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01121, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.06896520103328, "model_name": "gemini-3.1-pro-preview", "output_tokens": 732, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:02.511253+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b918fa0d-0989-4d8b-b637-d5e9e5692542", "result_sha256": "9306582a219e49a2e1848730c22b266af470279e65274d362d201d0b71743996", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006878, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 4.653898206073791, "model_name": "gemini-3.1-pro-preview", "output_tokens": 371, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:12:19.531248+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7d5fd89d-1dbe-4acb-acbc-80bb67ae2ca1", "result_sha256": "2d87c398be946a4b3b276ce0b8927f1fc36f963eb58e841784450a89859b34e0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008798, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.056827415013686, "model_name": "gemini-3.1-pro-preview", "output_tokens": 531, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:11:37.523267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "83f72620-c387-4234-a843-cac6c1a5c35e", "result_sha256": "f3c3437284f8f597061a7fb8edc8bdf9e2b5ab74ad76a1ac99644a4bc1a9abcd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01247, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.248653414077125, "model_name": "gemini-3.1-pro-preview", "output_tokens": 837, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:32.661680+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4e8fa83d-d768-47ca-ab27-616e93362b23", "result_sha256": "a4f783a6239df169c52e089bfbef8298947e9be66489707f223800bcd26b1b90", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00713, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.066797427949496, "model_name": "gemini-3.1-pro-preview", "output_tokens": 392, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:20.506687+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "5d41fca7-7731-48ee-a733-9670ab00a3c0", "result_sha256": "fce7b8b2cc5f2bd65dd5a4824935387226de8addb4528a2017eabaac7482d7cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008642, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.9525555149884894, "model_name": "gemini-3.1-pro-preview", "output_tokens": 518, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:28.495715+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "8b883e32-8571-4302-8ddc-8114f4c3fa42", "result_sha256": "29b9cc6d2db6a281db2bcadddcd5ea008b3412a7e0aa789e1656b529d54fbb1c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012686, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.381633977987804, "model_name": "gemini-3.1-pro-preview", "output_tokens": 855, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:15:29.865006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2111c171-43b4-457a-9173-978be921bf0c", "result_sha256": "22812faad933f51dc123e0d34da9899274f1733116f59c0936cbbfa2891eac75", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007454, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.250315281911753, "model_name": "gemini-3.1-pro-preview", "output_tokens": 419, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:54.790460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "303f5778-a0f9-4b4d-930c-ce8e3d0e7b43", "result_sha256": "f2328080d7fe0ed4c447273ffcb8b44e9e3cdd369823f94a4f6f0bf257f78b45", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009074, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.645248732995242, "model_name": "gemini-3.1-pro-preview", "output_tokens": 554, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:25:58.921163+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1ad07dd-dddc-4dba-bbb1-7bd3bab4ed33", "result_sha256": "07a1d88c7c7ef87b3387f7492c3d439f8faee99491386443501cb8ec8b2286a8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 4.032184044015594, "model_name": "gemini-3.1-pro-preview", "output_tokens": 282, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:26:52.497324+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e2338ec-b9d3-4760-ad43-4be65659323a", "result_sha256": "a2e703801987724d95f2fa448bcf812deab3d4e8e0690e8d99615cca58da2d4b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008438, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.706884339102544, "model_name": "gemini-3.1-pro-preview", "output_tokens": 501, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:51.732188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2677a7c7-20c8-4625-9935-15f5c1c09620", "result_sha256": "ee05c80056a2e358099dc053cbf73dddff16e2b59dc93cbbaaeb5e3e0b9354f8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007898, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 8.6101007029647, "model_name": "gemini-3.1-pro-preview", "output_tokens": 456, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:43.299317+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c02e91be-f770-4a91-88b8-f6ce5e6b7142", "result_sha256": "be1b2dd8df6640223038de8a7fd28f6d0e6aff5012902e58b83baa34460f74ce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008918, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.814857040997595, "model_name": "gemini-3.1-pro-preview", "output_tokens": 541, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:27.774798+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f93fd3a7-e47d-42f2-9cc2-9c6495ff3203", "result_sha256": "7777dc9a78efe16782839f923a399b21335a62d55fbc38bf6082224c06f6641e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016526, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 10.486833870992996, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1175, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "google", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:23:48.772085+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "fd974454-7a98-4368-a4ac-7a30f0f5f57c", "result_sha256": "bc9f24d2aeb462b7c0d4a051e53fdc1ed8227c7a162ec677e076635385e2c66d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012278, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 7.57854313601274, "model_name": "gemini-3.1-pro-preview", "output_tokens": 821, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:30.737433+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "92418b18-f466-4222-a3da-f34df69a275c", "result_sha256": "4bae1b227a1f763ef6cba7827b954827a3b165233ef1a38351c85669f704c247", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011906, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.962874019984156, "model_name": "gemini-3.1-pro-preview", "output_tokens": 790, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:17:07.888908+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "18a9857f-8ceb-4cd4-9be0-8e72bb2808ae", "result_sha256": "10581d0d3e1595a124f4a25b2a3c77618a50c759a33fbb443c87efa1c11a739b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014342, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 9.516599971917458, "model_name": "gemini-3.1-pro-preview", "output_tokens": 993, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "google", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:21:12.135840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f02484b1-8cb5-40fc-a491-3f2ba9560940", "result_sha256": "a09657d2765b47f6f00dfc4b6f67314a294590b7d06bf03ddc60876ff048fddb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008438, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.96843675407581, "model_name": "gemini-3.1-pro-preview", "output_tokens": 501, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:02.056867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "fedcab1f-608b-4a4f-9aa6-cfc0c8f8a92d", "result_sha256": "5e1fa5d1b2d606800a82e7960c552da704a543e0f54fba104f0efc82ac0e4ef1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010442, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 6.798644572030753, "model_name": "gemini-3.1-pro-preview", "output_tokens": 668, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:13.809279+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "460f5f4b-a4c5-49da-8aa3-4a29c1279cb8", "result_sha256": "7a4166b039a9aec56ad12df879bdacd0f05a15c23fdc3649d145e132d232c553", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009674, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 5.910796329029836, "model_name": "gemini-3.1-pro-preview", "output_tokens": 604, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:38.646582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "aafd0f45-4a8a-4238-b5b4-825bf111446c", "result_sha256": "fcee382a51f0590b60e3a724d3800f1dc4cf373919b7c238edfaafa7575493e8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014594, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 9.843833927065134, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1014, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:02.954671+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c4b022b-3ea5-49d8-861e-691c1591e6b9", "result_sha256": "6aa5fb97b55ec2c5268a6397ed5e80cae9b6f355e92545e9c3f58063ddefc5f4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008174, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.5469445899361745, "model_name": "gemini-3.1-pro-preview", "output_tokens": 492, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:13.853544+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "e9bb398f-0f61-48e5-9ef0-042e852c6813", "result_sha256": "0134e452a917cb0c0706066a8ac1922dfd2d96685b7231bf678026e61a9190a1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008234, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.230458915000781, "model_name": "gemini-3.1-pro-preview", "output_tokens": 497, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:57.214788+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "888aeb64-f931-4227-a025-4459b46e2cb8", "result_sha256": "5a61581e60bcfac764d3cf93e3a4d5ba47bbe3616316c6c49c17ed65a52b399f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008942, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 6.214299532002769, "model_name": "gemini-3.1-pro-preview", "output_tokens": 556, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:32.227676+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "20281b94-d007-4a0e-b574-0b48eb8c129d", "result_sha256": "0e112d42f93e85328f943aa122a5455a66dadfd540cdea95ef1974cc96c903a5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007802, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.493426387081854, "model_name": "gemini-3.1-pro-preview", "output_tokens": 461, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:57.746567+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "61db05f8-9189-4b96-8690-6eeec17984e5", "result_sha256": "2cd80721847a64da5bbf7316754e4bb1b0460bb6476bfaf620c130b93a6f332b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009086, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.959923560032621, "model_name": "gemini-3.1-pro-preview", "output_tokens": 568, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:04.857265+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "d89936a6-6da9-49ff-890e-c01561642fbb", "result_sha256": "a5a3e5bfac5874e9fce52e53323e951879798ec677846954c596d43336361b05", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00581, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 4.177867999998853, "model_name": "gemini-3.1-pro-preview", "output_tokens": 295, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:03.675572+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d914ba0-644e-47ce-afd5-cc0a99ce7295", "result_sha256": "4d587fccc65315eb1eeca713d0473ff5554fdd4ef8f51f28b0adcfded647d9af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008654, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.9708767760312185, "model_name": "gemini-3.1-pro-preview", "output_tokens": 532, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:23.413445+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "ab06423b-3637-4b82-9979-6aefa584edf4", "result_sha256": "41d33aaa0d93d64cf0305df915cd827c38e68b45fdce4beddf5a2a9169c0b432", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008042, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.349751467001624, "model_name": "gemini-3.1-pro-preview", "output_tokens": 481, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:39.690245+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f28a1009-c171-4d63-a7d7-1bf5113f2bc7", "result_sha256": "85992465a1c4623b247e98cd930062353dd48565e236aea0954f3a9cb703f683", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009266, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 7.297288144007325, "model_name": "gemini-3.1-pro-preview", "output_tokens": 583, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:13.729443+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a361b689-8344-42ea-887b-35708b5c4eed", "result_sha256": "21b98284264abf539569359c3d7a3ecf6644503e1ecdacdd61dc0e4eacbaaa9a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00803, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.298401274019852, "model_name": "gemini-3.1-pro-preview", "output_tokens": 480, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:38.130211+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "be030cdb-0445-4cbf-aac6-0468a5306db6", "result_sha256": "d88c8403d3b807e4e3dc0140bde938cf5c68034e45127abeff4499b8e9364216", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009662, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 6.150250214035623, "model_name": "gemini-3.1-pro-preview", "output_tokens": 616, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:23.807963+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7bfaf621-c735-4c7f-9fcc-2a9440f6961d", "result_sha256": "e7f5037766c84f30a038b56ba25c01d6eccfd88bfabe3e2c7ce75406f91a9051", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005762, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 4.478169625042938, "model_name": "gemini-3.1-pro-preview", "output_tokens": 291, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:01.016475+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "710add76-30c2-4f60-adf5-1176eaf5c834", "result_sha256": "cf8c1c840231a9289c535944206650f5e2432a2ed1892009f357eb49c0bd6bb3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00455, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.5501837680349126, "model_name": "gemini-3.1-pro-preview", "output_tokens": 193, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:11.848635+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "86ff042e-1fdb-44d1-89e2-7a97bdd8559d", "result_sha256": "6cadbc9f5cb9ca3fa81b5eee6c639b19d31ab3c6924b8e3aa76eda50ac27a567", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003818, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.247513381065801, "model_name": "gemini-3.1-pro-preview", "output_tokens": 132, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:26.181245+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "fa902c21-b80d-4f02-bfe4-5fd4a778ebb3", "result_sha256": "d264d7eb947708404e01360b37d48eabb4bfdaa0dae9076761a2d31a6437b8ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003434, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.1919428430264816, "model_name": "gemini-3.1-pro-preview", "output_tokens": 100, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:13.082624+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7013bcf3-9ec7-4204-8dab-13fb9943dbac", "result_sha256": "af66eb93b960de1ac967a816e81b345163aedc66a68411d2712df09bd2c8e6d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003458, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.2465716230217367, "model_name": "gemini-3.1-pro-preview", "output_tokens": 102, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:44.389927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3f4fa21-02eb-4a55-9c05-97de0c5eed76", "result_sha256": "636486f1cd104ffb0489749d197331cd8c3d54a3d2aa45900de877a29c4868c0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004214, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.399967946927063, "model_name": "gemini-3.1-pro-preview", "output_tokens": 165, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:49.253958+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "6ff36dcd-0098-4bd8-b722-2d479712e9c8", "result_sha256": "c1af0a4c34c497fb3ba3d2f43aebe49321d18f5c9234d32f8f8de975bfc3f54f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00611, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 4.06434810499195, "model_name": "gemini-3.1-pro-preview", "output_tokens": 323, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:17.377163+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a2fd3506-4917-4587-8d93-d03d01d55fa2", "result_sha256": "33331d493cfde3a2f93e310fa57b9e6710bbfa986d9eeeaf31f274cdc2f7a6a9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003218, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.8672300160396844, "model_name": "gemini-3.1-pro-preview", "output_tokens": 82, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:57.160747+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "0dcdc0fd-ea51-4e41-a04c-e1a0de9a15c7", "result_sha256": "641a5d1fbce2c8f5236038d7408679905965ffe52728a2d30e6414de4e6746c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003482, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.672649472951889, "model_name": "gemini-3.1-pro-preview", "output_tokens": 104, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:00.925722+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b2f3a69-056e-4e45-b86e-1dc9ff0b633c", "result_sha256": "a10de65e90086d24aa75ba47d49c1144072493349d630cced7a2d53137c1aea1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003614, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.409564721980132, "model_name": "gemini-3.1-pro-preview", "output_tokens": 115, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:40.204902+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d639378-909b-42f7-b079-2a2e1ee7a4c7", "result_sha256": "3e7ffd546e358a0606c48248af11cee3ed221df933c7100e416fa316a0f553f4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003698, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.5228998919483274, "model_name": "gemini-3.1-pro-preview", "output_tokens": 122, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:31.853530+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "765d35bf-2def-4f31-8013-94aaf4601135", "result_sha256": "9c3e48811949833476539b45b44a4f6bd045eb03b7d4cb543114bdd83250c4a2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003866, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.0638955559115857, "model_name": "gemini-3.1-pro-preview", "output_tokens": 136, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:35.423981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "3f5cdf43-24e5-4a9b-905d-5b66ed7113ee", "result_sha256": "7b9898c680e97a3fdec055a71f23c55ca40448414fb3a97539445e7a4b713e96", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003482, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.0122045759344473, "model_name": "gemini-3.1-pro-preview", "output_tokens": 104, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:23.125696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9ba72db-8ca7-40eb-aaa9-52a8609a1fad", "result_sha256": "9e4407dffa76fe4a266ac3f158456a5cabc1c18c8994ec5c3feea4176316ced3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011034, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 7.931975774932653, "model_name": "gemini-3.1-pro-preview", "output_tokens": 731, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:10.896931+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d071f50-7b0e-4c21-b572-2e61781345bd", "result_sha256": "a44772c5614a84679fb2460f746cd3e0487922a98d85158c811e0e53240c9e49", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01227, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 8.085181782022119, "model_name": "gemini-3.1-pro-preview", "output_tokens": 834, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:37.383928+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "bb8f8285-3119-4dbd-8be3-09e3d7c8fa08", "result_sha256": "2d03eb423a777dcd1c8a304a4de5b0852f9155fabcff203571c5fddec86bf3c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010446, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 7.216853079968132, "model_name": "gemini-3.1-pro-preview", "output_tokens": 682, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:18.769559+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "aeb57fe7-5377-46c0-86da-078428aa11b6", "result_sha256": "4dc002286ba5fb45467b8737dd26fc1daa2e0c5d24b28cf75c645c77ecd2474e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014922, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 11.399693761020899, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1055, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:39.170378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f595de9d-764d-4ba3-908c-ec8340027a66", "result_sha256": "b9fe118e3dba83aef332d7fbc7b29a2526101d036e51ae53efbc5b029d25cfa0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012966, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 9.6022658420261, "model_name": "gemini-3.1-pro-preview", "output_tokens": 892, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:36.843860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "03b35386-d9f5-4e5b-86b3-8d598ca19c3b", "result_sha256": "8ae67c024c6403d029f00c675f0c323801f3d99bd83700db19c7470e01dbaebe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00993, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 6.776845525950193, "model_name": "gemini-3.1-pro-preview", "output_tokens": 639, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:44.987667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "912b63dc-569e-4076-bbf8-33711c59c53f", "result_sha256": "d8c0cd306056d01bbab8618113334a2cef0f67c6472314242906f907754d7546", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01335, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 8.3414646780584, "model_name": "gemini-3.1-pro-preview", "output_tokens": 924, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:05.629292+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "e9623883-a8a5-49da-8de2-102be080a64d", "result_sha256": "aa8f783a898ef2f16d04e6a7ba19e534aaff359e459db18d5bcef26ec1fc927f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.020094, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 14.018715235055424, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1486, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:38.413337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c77ac3a0-3c27-4388-8eec-c8199dda1b5b", "result_sha256": "8f4abd2c3d2c6071d2eb276f213cf3fc62db76a478e1e9d952cd656a82cc6833", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.033642, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 24.531029382022098, "model_name": "gemini-3.1-pro-preview", "output_tokens": 2615, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:02.103858+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "3868e6a5-e376-4006-b3d4-a0648ed2abae", "result_sha256": "34c94e142448342c4ed9088d92c51e4456f47db54c82a0c9c5d1b05f25c5da0f", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.015378, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 10.138744972995482, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1093, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:25.768012+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3957545-0041-4ad0-a39b-00c256a4c1e0", "result_sha256": "b1c0410ebb8d8de563a64daf620b08a1b280dfdaa833bc4391eefdfd357ea4e0", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01449, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 8.885141520993784, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1019, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:16.984249+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2ecbf3f2-deff-4550-8939-9ae8eba75e39", "result_sha256": "afe3e12769a5eb810ca17b19b4e871e426cbfc9c9756a1facb514439bd27b804", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00861, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 5.665666086948477, "model_name": "gemini-3.1-pro-preview", "output_tokens": 529, "prediction": { "choice": "B" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:24.970982+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d1d090b-5114-4981-8a92-af6b4fd7d74c", "result_sha256": "8e93dd75e0a0b6080611bcbc8f9ce4f6126353510a01fbe61d2c7c5d27eb22be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003698, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 4.103257143055089, "model_name": "gemini-3.1-pro-preview", "output_tokens": 122, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:07.846389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "15b85051-eacc-462c-90b3-4aae6c6e007c", "result_sha256": "fdeabb3fd5bca75e3f89d8f81c66599e171b7e732279486e7061fb8e8e21c95c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003494, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.3431399040855467, "model_name": "gemini-3.1-pro-preview", "output_tokens": 105, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:14.023289+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "83c1bc65-ed66-4e47-a369-d0b88d62ba77", "result_sha256": "20ab8c37b01fc2da25405459675ce7cfc43f41ec5c797dd1fdd9a6de2a4d7c26", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00359, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.986867479979992, "model_name": "gemini-3.1-pro-preview", "output_tokens": 113, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:04.640429+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "616a4475-a104-456f-93d6-ed0356962a36", "result_sha256": "1fca3d9a22bf6cc2fb4e2b5bc047dbc8cb71940dd5af78682a3f6e1f249e50d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003638, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.7974426470464095, "model_name": "gemini-3.1-pro-preview", "output_tokens": 117, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:54.373272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "334d1859-4505-453c-8b45-a47a0b488b4e", "result_sha256": "570bc11a1f9dbc64b06f74bdf15b4dad19124809d22ef47fbaf038a269f0a4dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003518, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.002968106069602, "model_name": "gemini-3.1-pro-preview", "output_tokens": 107, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:33.206655+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "158901ee-284e-40ab-98a3-a566a7d9311f", "result_sha256": "c059b7c143d788f77b6f65b76b2454c84307d750d64bf1e71cdc1785b578b622", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003758, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 4.114626103080809, "model_name": "gemini-3.1-pro-preview", "output_tokens": 127, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:46.741440+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d121f77-972a-48a1-ac8a-177405eb7d5d", "result_sha256": "7e1913f3cdffdff3bf50b1e88eb409ef127aa9436de730bdd7048be6344a24b1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006494, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 4.731778824003413, "model_name": "gemini-3.1-pro-preview", "output_tokens": 355, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:55.283701+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea4474c6-9475-493b-9c92-cd5f7ef2a6e2", "result_sha256": "405f0fe2680f2dbd0cb04ea6ad84af6fc44f5046cf97d284a64fd69c0602d8f4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.3862357520265505, "model_name": "gemini-3.1-pro-preview", "output_tokens": 116, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:06.554750+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c2844aa4-1ae4-4a7e-ba38-ccc832311ba4", "result_sha256": "97930c6b343bbb606620302eb7a0719aefa14df946916ff2493aec6242fa1a4e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004178, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.6916412389837205, "model_name": "gemini-3.1-pro-preview", "output_tokens": 162, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:15.541160+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "621bcaf5-f46a-43b2-a33b-c5ae73c9333d", "result_sha256": "e5234d2dfaec0f75956ddd5c6a385c03a0eb6ae73a783bffe33425096687de18", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008078, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 6.038641278981231, "model_name": "gemini-3.1-pro-preview", "output_tokens": 487, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:31.596639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "056a57fe-4194-470d-ba78-2205090bab4f", "result_sha256": "ea1d099b6a16d67494de7ba787a25aaf7ef7c7631533056649f8b8cbcc81cd02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00323, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.594518308993429, "model_name": "gemini-3.1-pro-preview", "output_tokens": 83, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:54.296531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "876558d3-380d-42f3-a0a5-450d5d73ec83", "result_sha256": "2e833a1acbbba8e6022e1db8d1b53255934d53e34603057d7c6f5e487724eea2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004226, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.950854792026803, "model_name": "gemini-3.1-pro-preview", "output_tokens": 166, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:53.354885+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "9317cf55-2ea8-475d-8a03-80e99a59ec0c", "result_sha256": "e09cbc40826a3475f6ae732ae23ed9ae9e9bbd9cda097c7f045d20727dd88118", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005136, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 6.060403982060961, "model_name": "gemini-3.1-pro-preview", "output_tokens": 238, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:59.520201+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "d4e283af-13a8-47fb-901f-9878bf173250", "result_sha256": "f22f15f331b45e58dc91f3022b397f80c6a50a64289fd4b43a396b53ea00366f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005076, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.990112956962548, "model_name": "gemini-3.1-pro-preview", "output_tokens": 233, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:12.290708+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7ade78ce-b052-4224-b7cc-ef99c81b9be8", "result_sha256": "b88757530ba51c0c189a15016d3d146567de04f3572e7691b3c234efc9203d86", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005556, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.658496511052363, "model_name": "gemini-3.1-pro-preview", "output_tokens": 273, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:08:45.627782+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2063b8e1-b727-4f09-bd20-c9d356f34211", "result_sha256": "ae84e183b82bd8be82d468b49aca0053eede89b824c8eb8aab1b714e293ad8bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005544, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.116866817930713, "model_name": "gemini-3.1-pro-preview", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:07.054398+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "9826de80-99a8-4d82-8137-9d6b648374a7", "result_sha256": "bde5b64ec317d6a4b155fcbb9587e7a23207e4d808b70e8dad2063f61fcb41a3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006348, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.51203180802986, "model_name": "gemini-3.1-pro-preview", "output_tokens": 339, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:28.888100+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "60412f28-f8fd-497e-87cc-722387025fc9", "result_sha256": "8a5509a38d75c8137344a78ed11c04ac18d65fd79732ae8f03b9fce41f73ff13", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00384, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.5140571910887957, "model_name": "gemini-3.1-pro-preview", "output_tokens": 130, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:21.224761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "89f0302f-31f0-4617-b3ec-7eb06b205b2a", "result_sha256": "e16eec9e85b6022b982a10eeab6a1f233d931593cb8dc767a7111657912c28c7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004536, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.731966313091107, "model_name": "gemini-3.1-pro-preview", "output_tokens": 188, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:39.584733+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "15856f39-d72d-49b7-bf8a-17b23d9ddf49", "result_sha256": "f1f5ddc0aeb48bbdc0d0b4577e65f65daaa561923385ad68d4e0c7d42869b551", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004704, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.807069333968684, "model_name": "gemini-3.1-pro-preview", "output_tokens": 202, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:42.466382+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "77f98d19-3ea2-4dc8-b1df-d0f1a7c282b8", "result_sha256": "ffa896132cf98b40dffc913272bea7a8cb59d7821bf12dfe623483c743267529", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003768, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.340015136054717, "model_name": "gemini-3.1-pro-preview", "output_tokens": 124, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:58.675600+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7f649d36-7fbc-4773-8611-22c80071f864", "result_sha256": "27693f77294fa88ebdb9490ba77f08acc123f6f396101950f1d42167159a5986", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006804, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.707865001983009, "model_name": "gemini-3.1-pro-preview", "output_tokens": 377, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:09.234387+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "d2e012d1-8753-48c8-8928-3d225713d54d", "result_sha256": "588b65255e4ce3263a9adf1c7cbdff28e5b87c47794916b94faca6af8bc2b757", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004464, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.321522808051668, "model_name": "gemini-3.1-pro-preview", "output_tokens": 182, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:30.201483+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "62e768b3-a988-48cf-b180-daa511563fb1", "result_sha256": "5724b21d0ac4a5a4df94cc76ffb8a1afa269469fecdd22e73c2bb587da653363", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005892, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 4.683150792028755, "model_name": "gemini-3.1-pro-preview", "output_tokens": 301, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:20.420067+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e03b5ee-6e9d-4d72-8a4a-64da2796ddc0", "result_sha256": "9aacd675af32298189abc0a4ea1cc16ae27843ebc071f0d2a495d61b63c20eb9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010326, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 6.693077166099101, "model_name": "gemini-3.1-pro-preview", "output_tokens": 671, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:48.744860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "0d9d54f4-e5ad-45f7-8e95-53f3f1029dbd", "result_sha256": "355944b7b7a4a486bd513d3f61e1ef6d23c41ffc773746f10a3b27615db5a7cc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005214, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.369887802982703, "model_name": "gemini-3.1-pro-preview", "output_tokens": 245, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:17.300730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "9bbf90a3-51f6-4985-b0ff-2308f88c0f2c", "result_sha256": "4fc3cb31190a866e6c26873cf8f827e0030ab9c8bc1b3c4dec2d4c41c5762752", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007446, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.460437801899388, "model_name": "gemini-3.1-pro-preview", "output_tokens": 431, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:33.059141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2768cacc-28b3-42fa-900e-74ceedda0520", "result_sha256": "7ef062a59bfc9fb3481c140f34f2df34663e9e95c4d9a26d4c3902aee35a80d8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00771, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.687882098020054, "model_name": "gemini-3.1-pro-preview", "output_tokens": 453, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:15.498366+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b16b5bca-4a1b-4d69-82f1-d44d915fd8d3", "result_sha256": "08058c9ef476c0cf9b3488d1385a1d2d6e6e36cc1896aa1a530aa4e0a5c2f165", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00645, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.716169400024228, "model_name": "gemini-3.1-pro-preview", "output_tokens": 348, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:10.656761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4ca56d86-9651-4941-b836-0ae0eece6c31", "result_sha256": "355529c73e0ac3b2d4ba316a64248bbc314d72d16ab73e6eebd615dc7081b0b1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00567, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.004715334973298, "model_name": "gemini-3.1-pro-preview", "output_tokens": 283, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:23.207299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c406510d-5bdf-40f4-b215-c306a7007c57", "result_sha256": "eae2d48ac46f85d9cff41e0c92d10db2cf211101b379f50b9b5ec4e7a2b95ab4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007554, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.58156154199969, "model_name": "gemini-3.1-pro-preview", "output_tokens": 440, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:47.639840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9a907d5-36ce-4e30-b568-fe6d3ca61533", "result_sha256": "6d9ee077bcec969505a78aba4c4c44d50a3b11d4f261151a248440f840494b25", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00645, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.135882672038861, "model_name": "gemini-3.1-pro-preview", "output_tokens": 348, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:34.278056+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "e8a9269a-730e-4a71-a0a1-d5881def1da2", "result_sha256": "ff32ee25e01ecddbd7854baac5f590be6980b2d40893c2ed3890f05eb094f834", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00639, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.546405446017161, "model_name": "gemini-3.1-pro-preview", "output_tokens": 343, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:49.655498+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "d07165fd-5f0c-4a79-8e10-83d70f85944a", "result_sha256": "d746bb84c93c66370166f3a3acc1f1fba9602de7c4b7fd7324f5f816da10124f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007338, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.350948475999758, "model_name": "gemini-3.1-pro-preview", "output_tokens": 422, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:26.468091+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "1fd7da2f-ccde-4e56-ac1d-a8f4c0dc9e52", "result_sha256": "a5b75825465ab6a2d95ef43f8b116379e5100f5bcfe8ef3a9cf97a1b83e09fb2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006942, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.454103809082881, "model_name": "gemini-3.1-pro-preview", "output_tokens": 389, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:28.643874+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "fdd6a016-b6b4-46d9-9f7e-79f011cb43d2", "result_sha256": "24e019e79a4d0bba856cec022a6325759c3f7091ab9ed4f1afea06b1af654e59", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006174, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 6.0739705869928, "model_name": "gemini-3.1-pro-preview", "output_tokens": 325, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:49.373121+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b9b478c-d207-460d-add9-41d3462cf678", "result_sha256": "e27b3564f8427ddfe611e98fc48462b330f6b0f203e68c5bcc885810af40f2c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005754, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 4.808205430046655, "model_name": "gemini-3.1-pro-preview", "output_tokens": 287, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:27.090812+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d9c6c0b-f7b2-46bc-be79-619280ca58e8", "result_sha256": "04afb4e13085e53fc93b17641f593193f90e9b5429419e84b83151f1b5ad6506", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00795, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 5.29071589501109, "model_name": "gemini-3.1-pro-preview", "output_tokens": 470, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:08.814632+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "beebae40-7951-453f-a974-02680f07b117", "result_sha256": "3290dcf179670128db99b6c1167487cacb2d8d0ac2519deb7b60cc09574c8e95", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009006, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 5.756578528904356, "model_name": "gemini-3.1-pro-preview", "output_tokens": 558, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:27.357258+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "ccd12125-4b44-4a99-ae0f-335abd52caab", "result_sha256": "0008b5c02f08054dc2a714e1e1b5ee70e028e8a61f3edfddcd6cc37a4e865fd3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006822, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 4.862825941992924, "model_name": "gemini-3.1-pro-preview", "output_tokens": 376, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:42.804318+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "5a335387-1243-4ab8-9e2a-41d13b42865b", "result_sha256": "0221e8fa290929e314d312b01672f6465cb1e6333f1dcefbe792f5c9f8b3dbf5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.023154, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 14.085953654022887, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1737, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:00.367872+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "07e08df9-af12-41bc-84a6-058483dbf8d4", "result_sha256": "0061aefaa6440edeabda24befafd965045fb290d359a855aa70686d39394becf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.014466, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 9.596139468019828, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1013, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:19.273410+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7c2ac5ab-9a94-402d-aa6c-e6663a8c044a", "result_sha256": "5ed3ac2f217e723c52dc88c5be8873989970b8369cad7d01e7481e42aba34d7f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.03207, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 19.9317276700167, "model_name": "gemini-3.1-pro-preview", "output_tokens": 2480, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:03.488114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "71f2262b-c1c9-43e5-ba20-4b5a70bfdac9", "result_sha256": "b1c4f7cb3a57aca092453d602a126139baf7161b2304a5a85d9b6b33e9881404", "result": { "correct": true, "cost_estimated": true, "cost_usd": 0.072178, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 9.201464507961646, "model_name": "gemini-3.1-pro-preview", "output_tokens": 893, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:45.997289+00:00", "request_attempts": 2, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 1, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e6ac51f-135c-4570-948c-3a58a50912aa", "result_sha256": "909e0adbe62d483100342250eb12c8895a4dbe39588e18c9336990ba754ef984", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011922, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 8.227008865913376, "model_name": "gemini-3.1-pro-preview", "output_tokens": 801, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:07.713969+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7dd68888-711d-487b-8f0d-0dc7baff059f", "result_sha256": "63789ea142e69c5a00dfbb72abfa0574dc4a2e3f25cefdd55fbe14bd10c9a35d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011058, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 7.557253677980043, "model_name": "gemini-3.1-pro-preview", "output_tokens": 729, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:19.048545+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a6bcc59-5ef5-49a1-9142-523fc7e901a9", "result_sha256": "e267573cd6e589b0301cff33d28bc017fe497b3a6cb7791f0472354545b83c53", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008586, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 5.478442589985207, "model_name": "gemini-3.1-pro-preview", "output_tokens": 523, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:21.596292+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c046b343-8e35-40e0-a095-7e8ba5757f37", "result_sha256": "0717686971f6166e75a732aea21cb0105c598cd199e05cd5beaf866fff8a7883", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011898, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 8.658658276079223, "model_name": "gemini-3.1-pro-preview", "output_tokens": 799, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:43.227455+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "9959c831-7a83-44af-a761-6bcb7ab9b4c4", "result_sha256": "c7e817b1a572b20b06879adf4dc8cb01af81edf434aab92676d96d8660cd3713", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003366, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.6129144199658185, "model_name": "gemini-3.1-pro-preview", "output_tokens": 91, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:56.341688+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c14210c7-bbbe-4781-b3a1-6195ecf4c8e8", "result_sha256": "e4e81dd5cf3ae1b693d18dd42b6e72f35c19fd03ec6199f3c4c25882515f64ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003714, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.927906552911736, "model_name": "gemini-3.1-pro-preview", "output_tokens": 120, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:03.818863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4217ba3d-72bd-4ed2-be0b-8e4db8676b32", "result_sha256": "b81ae1fe5e98656fbe3c2facb101bf34671153d55512f70608da9873c1ba72d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004926, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.059048336930573, "model_name": "gemini-3.1-pro-preview", "output_tokens": 221, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:25.497566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "9b9e0e39-25e2-4a67-9318-94c4fdd9ddbf", "result_sha256": "4fb79e301de1a5e6e44838d7348411deadce922b1b152ce0b3d91b022fbeff1e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003342, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.734972445992753, "model_name": "gemini-3.1-pro-preview", "output_tokens": 89, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:57.082689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "0c90db1d-62d4-4d66-a806-dd93434ec7d1", "result_sha256": "0705d922cf142b05eee61ad2bc5033c2abcf7472d3f11fdaaaaff0060f4daa75", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003162, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.2554974919185042, "model_name": "gemini-3.1-pro-preview", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:49.323787+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4e6b6730-53da-4d87-a96c-3f9f2faa48ee", "result_sha256": "227f1235546717d549f93d3fdb25912527f080041830c5dbf31d46fad5aa1ce0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00327, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.6484703819733113, "model_name": "gemini-3.1-pro-preview", "output_tokens": 83, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:22.218181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "bd6dd7e5-dae8-4587-89d5-d970cd092258", "result_sha256": "1efceab717819ee5ecd82133669ed2d20e66082abd9020e34b5bd071a9a0a178", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003354, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.7921555089997128, "model_name": "gemini-3.1-pro-preview", "output_tokens": 90, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:29.135424+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "53d3ad5e-c1aa-431b-99d5-7134c3f9fbaf", "result_sha256": "b00ace5c800ca16209d41e202b96ba46ab598c6cce12af349e4a872ba7e51a67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003906, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.2383609699318185, "model_name": "gemini-3.1-pro-preview", "output_tokens": 136, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:13:48.086657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7579fdd5-8774-45f3-b6da-8e6b3048daea", "result_sha256": "3a38add994173446c23f1c51e8bc6e94424814aba0fc9670835d12ea386df158", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005802, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.006356008932926, "model_name": "gemini-3.1-pro-preview", "output_tokens": 294, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:12.892843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "52a53039-9b4e-4158-9e01-2f6158912f88", "result_sha256": "89f10b7db733ee79282fcefd0704c50fcb3e8003d1edfd868ed5beb741653e2f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004974, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 5.395406601019204, "model_name": "gemini-3.1-pro-preview", "output_tokens": 225, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:17.852193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "17126440-3fba-4607-b5f9-88af648130bd", "result_sha256": "375f5c9dd3d036673273ce7db03d149455ade828a5e4275decfad20f11808cd1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005358, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.913284022011794, "model_name": "gemini-3.1-pro-preview", "output_tokens": 257, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:23:36.614811+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b11d40a0-8f59-40e0-b493-f4af0882e42f", "result_sha256": "dc69fd9df1e1b1e4ebfbf98a53c17d2ec792b917b9898be94e5d8f4209e021d5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004146, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.2766582779586315, "model_name": "gemini-3.1-pro-preview", "output_tokens": 156, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:14.718527+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d7c6741-6e51-435a-89c8-d73548e69514", "result_sha256": "fce3a78de8646b38980604776e17b7d44f980f04bf687df4dec4433e90ffc611", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003666, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.794336663908325, "model_name": "gemini-3.1-pro-preview", "output_tokens": 116, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:48.440257+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a6d7c91d-eaa5-4032-af96-40d19973da67", "result_sha256": "6bc970ed28465191ba7a95bb0c15072f39473fcb512a4980c593296ada86fd26", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003438, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.1597562370589003, "model_name": "gemini-3.1-pro-preview", "output_tokens": 97, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:36.341518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a829890f-63da-4deb-958a-6e2a94fc09de", "result_sha256": "07f9b8ae4f67bc1a6f383a680e90190ed429bd5ee68704a05636421a1a1f1e21", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003714, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.7559834440471604, "model_name": "gemini-3.1-pro-preview", "output_tokens": 120, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:41.990159+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "5f0d3789-7ddd-488c-916b-8e65afb1c645", "result_sha256": "72f38a86d1cf5315eee6bb9839d53538e06f0d74d7f95a8c9242f1f36fedef19", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003762, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.0284659470198676, "model_name": "gemini-3.1-pro-preview", "output_tokens": 124, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:35.509220+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4332ea3b-cb01-434a-b3ac-ffe3e63f94d2", "result_sha256": "0ef394481c7e8805dc0a2b98bc432261b2d581cb8e3f024b082f4b0f35b0478b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003354, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.875242267968133, "model_name": "gemini-3.1-pro-preview", "output_tokens": 90, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:15.423706+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f39776f-126a-461d-b543-8da8639e81cc", "result_sha256": "a839dd83deb0b9ac0f498d29eacc04896298eec6253159bf8424ac55035577a9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003486, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.3395706430310383, "model_name": "gemini-3.1-pro-preview", "output_tokens": 101, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:17.794207+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "99eeca5c-32a4-4187-9407-603d26938b62", "result_sha256": "d117b8a5e575574133c02509b067be369ac317296fc90008790a4882e66f15ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003726, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.6575521510094404, "model_name": "gemini-3.1-pro-preview", "output_tokens": 121, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:56.979411+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "bad4e428-f718-4d42-9cd5-f923a624f0e3", "result_sha256": "bfc0154f449809eebab4af653c6a73624940e5669b9f3c9a11cd8d5873a2dffb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003378, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.9457292270381004, "model_name": "gemini-3.1-pro-preview", "output_tokens": 92, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:48.499209+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7e337e5-e9df-48c6-a289-fac0f0381d26", "result_sha256": "0271011c0ba589040595435db56cff41d90b1d2cf3982212381dc5f16d5da627", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004134, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.386068621999584, "model_name": "gemini-3.1-pro-preview", "output_tokens": 155, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:31.729582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a0c94f3-3d55-4be1-9394-615c4115ba44", "result_sha256": "1bbc538523a7cc64d3ff14606ccb59a35e2b8cc3c3c65f9a44fd9eddb4174d4f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003642, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.2458251040661708, "model_name": "gemini-3.1-pro-preview", "output_tokens": 114, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:07.746109+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2655c373-161b-49b3-af8c-7838b890f965", "result_sha256": "a5795223d2e14fcd1d373873e4586aec76529f40247f676770b008a42f753465", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004062, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.4672219670610502, "model_name": "gemini-3.1-pro-preview", "output_tokens": 149, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:40.585389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "9bc63174-5d57-4596-a459-be09204f2c44", "result_sha256": "4900bc39345aba9bf7b99491c3b1ca2ca450b5bf211d69f7915fe36ee4b7ecc0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003978, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.844536190968938, "model_name": "gemini-3.1-pro-preview", "output_tokens": 142, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:08.579319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "3995cfef-2fda-4dd1-b708-84c36bbbcb5f", "result_sha256": "021fb49705e9e6bd085723e7ebf89d6f184429c80deb0003ba95e30231c2de68", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007616, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 5.348469171090983, "model_name": "gemini-3.1-pro-preview", "output_tokens": 443, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:27.010774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc443393-9c6b-4948-b970-91d8e04aa6c8", "result_sha256": "2f3a327990afe4014091548cf785f40165cd85465802b7390812b6eb76fab9ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.677944949013181, "model_name": "gemini-3.1-pro-preview", "output_tokens": 125, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:31.997470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0d39f03-1aee-4b15-89d8-4ea54399cb14", "result_sha256": "446c053fa6bb6153169c2ffe3923df49b585fcbfca1a5a6b3c4d52995f4f0471", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01226, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 8.402571517042816, "model_name": "gemini-3.1-pro-preview", "output_tokens": 830, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:36.869960+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a90ac467-1449-40a1-b309-da42d8073020", "result_sha256": "3c5f167fe27c044520261a2d72f31d58593326e838189286e5cf1d1aae7f648c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005528, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.312374165048823, "model_name": "gemini-3.1-pro-preview", "output_tokens": 269, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:24.588935+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "92645036-36d5-4cf7-bab8-5f0ac6db827c", "result_sha256": "6c1be259772890041f919b203e0da5cde289e0fb08fd95c2c419b947ed8928ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005804, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.865882856072858, "model_name": "gemini-3.1-pro-preview", "output_tokens": 292, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:06.992248+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b9c37474-efca-4c10-99fa-d5fd2142981f", "result_sha256": "dbf83682a3017a58485f83857bf0d2af8f4a0203842b1fc111c9488eaeff29e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003908, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.601071208016947, "model_name": "gemini-3.1-pro-preview", "output_tokens": 134, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:05.495396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "69300f4f-32a3-46f4-944b-cfb1894c74e4", "result_sha256": "db3e4be3f3b98a685879e67567938877d736c64e249ce3e7d219f6b80b76e328", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006176, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.9550175149925053, "model_name": "gemini-3.1-pro-preview", "output_tokens": 323, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:37.653406+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "8f88f4f3-513e-4d7d-bb7a-62f11859ac77", "result_sha256": "0426bbae3f3681ed994d2b0c17db184f6d696086087311193efacc8c7206cc1a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005576, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.7415951860602945, "model_name": "gemini-3.1-pro-preview", "output_tokens": 273, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:26.850268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "0d63e639-6e01-46ce-abf7-97aa25f6ac86", "result_sha256": "b9a91094a7c58d00d2ebea233cc80240c658b285fd768cd38c9f692486da988d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009944, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 7.052551407017745, "model_name": "gemini-3.1-pro-preview", "output_tokens": 637, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:09.254819+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "5c785fa8-519c-4b89-b81a-c02d1f8cc9c4", "result_sha256": "a75691de8636747fbbc0d2d730e7dcda52095970b61e5eea9fe1aba050b1023c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006512, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.359754362958483, "model_name": "gemini-3.1-pro-preview", "output_tokens": 351, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:33.151980+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a55a791b-b65e-42bc-bad4-360cb0d4ec9c", "result_sha256": "f10e6caaec349350a50887cd234ff9467819d60c98d27f622835059a0af43ee4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007976, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 6.363318598945625, "model_name": "gemini-3.1-pro-preview", "output_tokens": 473, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:23.322815+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c2c4cb59-dfb9-4652-943a-a9b14bf8bace", "result_sha256": "c58ff76ec2a1d399084ac4f11cf1c49ade465fc51ba0932da97f2c00d34ceda8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005108, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.310344657977112, "model_name": "gemini-3.1-pro-preview", "output_tokens": 234, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:47.350172+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9ef7ba8-a2f8-4772-b106-86a7780b6de5", "result_sha256": "0e6e7f5c646e743710c097fcbc29fce1dba34740357fdbef77131af5d974ca16", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006104, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.660939791006967, "model_name": "gemini-3.1-pro-preview", "output_tokens": 317, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:22.159158+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "3569597b-cef1-4148-acfb-3df3df36520e", "result_sha256": "8fd1aa5222211390a5f18e4d903e584687939e60e2f7e9c337f21f3d20f015d4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00686, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 8.293730500969104, "model_name": "gemini-3.1-pro-preview", "output_tokens": 380, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:26.572674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ef293c1-de34-423f-bca5-0f721e73f57f", "result_sha256": "f7bf9bd124b26e64563d54f2fc6c81f31e937fb779bb13daf8459c53ed43d843", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007064, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 5.429854942951351, "model_name": "gemini-3.1-pro-preview", "output_tokens": 397, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:40.934602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4d7e2f7-23b6-46ba-9de3-46c3cf603e43", "result_sha256": "c39e13db3ef6b501656c7ed1b5ef86e2d8cc6a6a2aa98a9272a08a6f79763004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00704, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.450001001008786, "model_name": "gemini-3.1-pro-preview", "output_tokens": 395, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:10.258695+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "63a2ea9b-ccb8-45b2-bbb9-4ce4f782f6fe", "result_sha256": "e32ca508f77df371ca05244580d4ae6443806796da179e733211680546a05b47", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006692, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.863794052042067, "model_name": "gemini-3.1-pro-preview", "output_tokens": 366, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:26.846173+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "eaf5ed87-490d-4d45-8869-85d60af36369", "result_sha256": "d25a79b4d484510e79480c24ef0860da77c08c23bfa07bd0b14b09389b4285ac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003548, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.0901297860546038, "model_name": "gemini-3.1-pro-preview", "output_tokens": 104, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:33.460608+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "176aff27-ab35-4d8a-af02-d15dea53a9a1", "result_sha256": "047d7608e5b86e57e6b6758be815400f7acdf33bb986601714d1d94b6e0ecb33", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00386, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.9940413139993325, "model_name": "gemini-3.1-pro-preview", "output_tokens": 130, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:27.763289+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "43bc0a75-8f72-4103-af57-9384e8d96aaf", "result_sha256": "ebef80e7f18d5282316f4865952064e102ff8ef425729e9f40d0cbbaa5a4c481", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00584, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.657434228924103, "model_name": "gemini-3.1-pro-preview", "output_tokens": 295, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:15.710243+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "43eebf87-f093-4655-bf18-d0a490d65c47", "result_sha256": "0768f1da62508e1090465d756d0dbfbca810b21e3ea959d82470587d412ae5c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004388, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.6385376329999417, "model_name": "gemini-3.1-pro-preview", "output_tokens": 174, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:49.994111+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7514fbf6-d259-464f-a627-e68a0c9a50ce", "result_sha256": "2e24af5ee0046b99f4260e7469a2578cb94af2ef7d8d4259f4fc717cc01b4cc8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005024, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.557848635944538, "model_name": "gemini-3.1-pro-preview", "output_tokens": 227, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:01.775197+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7e13ac60-39af-4754-a418-17006ea9fc63", "result_sha256": "d0f5d2b89ec9feb1cadb8c18cdabbfde909f3efe9a3801b5c0ee1e6a8983ce52", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003944, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.206380732008256, "model_name": "gemini-3.1-pro-preview", "output_tokens": 137, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:30.314317+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "87f96299-ffaf-485d-b357-801067059ab5", "result_sha256": "3434a8eb5b8a4f95c9ffa085cfea828620385d37d742086e0c1ab4e27afea49d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008072, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 5.864690772956237, "model_name": "gemini-3.1-pro-preview", "output_tokens": 481, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:16:53.731633+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c54eb51-358f-48ac-a9be-49a7670256a8", "result_sha256": "d85bd81abc98f075dad491ea1e52d196203fc11e341bd4e02e1d3b57da1e9f8c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009464, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 6.065125687047839, "model_name": "gemini-3.1-pro-preview", "output_tokens": 595, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:02.794904+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a07d3ca-e02b-4200-87ae-cf02926d64d9", "result_sha256": "e034597aaed34573cfb2dd096523e8aab5147293794b62187afcb21c61c13ec9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.019472, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 12.169177787029184, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1429, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:30.080047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "0e33d3ad-0a82-444b-b54a-cac23d09618b", "result_sha256": "480671cdf364bfccd50e6dfb3f9fa38831371e24f01947a2e3e82b38bfae657c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007016, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 4.8384716040454805, "model_name": "gemini-3.1-pro-preview", "output_tokens": 391, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:16.148979+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8eb4bb0-e5d8-4ba0-8040-9b0c434b6aa1", "result_sha256": "fb3805cb40b1ce44132da2a58dfcd42556ea5c5d2d60f9bcd4c545965b35dd6d", "result": { "correct": true, "cost_estimated": true, "cost_usd": 0.06658800000000001, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 5.940043163951486, "model_name": "gemini-3.1-pro-preview", "output_tokens": 426, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:51.583136+00:00", "request_attempts": 2, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 1, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f32aa80-f0ec-41e9-817d-ed7c54bdc25c", "result_sha256": "e068e474c1e65e8146cab783d41e3b03f7b811f7b34f7c1e08318e295b069552", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00668, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 4.758484165999107, "model_name": "gemini-3.1-pro-preview", "output_tokens": 363, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:31.418983+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "ec2249ec-37b5-49f3-9ccf-f79a577d1281", "result_sha256": "c9c93f52419a671baacb4e9ea79226c9624f583ce273380d5c2d0d7b2022cc7c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01706, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 11.076936650904827, "model_name": "gemini-3.1-pro-preview", "output_tokens": 1228, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:42.197580+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2fcd8fe2-0deb-45bc-bac6-bdf40f35cada", "result_sha256": "e51957ee164aec49acc34fa060488f4c9d577098c91d6a3fddb93efe6f945c67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01268, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 7.660069461911917, "model_name": "gemini-3.1-pro-preview", "output_tokens": 863, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:19.832457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "d307335e-f23c-422f-9f71-b9ddaf5b56b5", "result_sha256": "5751fb00387f33873692f43fa8eeb663befb0c37cba3ceb6e584554244b2091b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010844, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 6.878681114059873, "model_name": "gemini-3.1-pro-preview", "output_tokens": 710, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:49.502195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "0987ae45-7e3b-4d89-ba3d-cacd68f318f9", "result_sha256": "d38235d13c8eab7324ae79f489d626e88b3d26a8b98d5deb35a1708b5176d344", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.028616, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 17.442417040932924, "model_name": "gemini-3.1-pro-preview", "output_tokens": 2191, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:54.512515+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "87ea8ea9-c94b-4fea-b967-da73c51b57ec", "result_sha256": "82e74ff6ee9a127ed2b25afd9e338ec70e28b7da10385b8273a42559b9308283", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011588, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 7.980633445084095, "model_name": "gemini-3.1-pro-preview", "output_tokens": 772, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:54.937205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b82e422-8c7b-461c-90ea-d4909aad1158", "result_sha256": "c65bdf2163931ab1c7519dda13b036ba56d33c08535959e99c356a8f651aab25", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.010256, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 6.335312629933469, "model_name": "gemini-3.1-pro-preview", "output_tokens": 661, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:38.845052+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "d11117b7-b091-43bf-a7e6-9ebfaf2a5b6e", "result_sha256": "71038171e462003b947f31463b0cf9016de5ade13df0b52e379f7f26c2cc7059", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.009032, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 6.518618904054165, "model_name": "gemini-3.1-pro-preview", "output_tokens": 559, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:06.915897+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0ba4ae7-4729-422c-8d17-bd2b73152226", "result_sha256": "a3a33267110944ecaf99914f2024f765b0a97a20f5677496b73f61eff185f6f4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008002, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 5.831615620991215, "model_name": "gemini-3.1-pro-preview", "output_tokens": 473, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:25:55.604311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b2c56e7e-0d5d-415e-be5b-0b1a891bc05c", "result_sha256": "8b42d7fd80679201fe6fb8a7a219ccfb1cd18816537d7ea4a70a45f7a72437c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005794, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.205903830006719, "model_name": "gemini-3.1-pro-preview", "output_tokens": 289, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:51.503539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "29fb646a-a529-4351-9443-d731bad001e7", "result_sha256": "8f51c4a95a90815e73300df3a9084dc9f00ba7623465bc8820e850148ef80895", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005794, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.481073799077421, "model_name": "gemini-3.1-pro-preview", "output_tokens": 289, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:50.706546+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e94ab9c-14a3-4dbe-9fec-d10dedd9e7e7", "result_sha256": "1366339b40faf019a163fb0811dd28f71cc1a7f719c45e476ed324ab8c48978f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006922, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.886587774963118, "model_name": "gemini-3.1-pro-preview", "output_tokens": 383, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:27:19.009422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "85ef6f51-7a2d-4d99-9ee0-f6067726f25c", "result_sha256": "8eb107815fa168ac8772c48cca64be4a0af175c13391f85d4076e15a1d4072eb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012442, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 8.2357761770254, "model_name": "gemini-3.1-pro-preview", "output_tokens": 843, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:57.615946+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7199c288-b844-4a9b-a2de-bb7db95dceb0", "result_sha256": "96d643267fda5d6012fc7def6d0efd8940b2ce837a3578f9a7d18bb8edf98029", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01351, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 9.234951923950575, "model_name": "gemini-3.1-pro-preview", "output_tokens": 932, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:47.104808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "26b87f4f-1a94-49fe-9b6e-ca32f670ba45", "result_sha256": "5d4934cb5c7c5074c517bda23308866b41e3d262b02e44da3f81f207606d0cf8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010138, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 7.3135864429641515, "model_name": "gemini-3.1-pro-preview", "output_tokens": 651, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:20.922069+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "89ca5b6f-b33b-4a27-a13d-62deefaf07b0", "result_sha256": "34303515318aa697e825679eee37590ba18e3edc070bd61f48af8309a54d1053", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005854, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.368368493043818, "model_name": "gemini-3.1-pro-preview", "output_tokens": 294, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:43.793657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "3be9e574-dca3-438f-a306-cbd86b076839", "result_sha256": "55c1bdf044545cc700516d59ee6490c43e9a73091c9b048537e2a7d116d32c1b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008278, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 6.0640768519369885, "model_name": "gemini-3.1-pro-preview", "output_tokens": 496, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:27.217082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "1061cdf2-ad1a-44ff-8531-9aa6816ddbc0", "result_sha256": "cecee2e3bc3da56682e80ca4ea60bd4b7ab41055a063e66fe6339900a2b7f215", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01183, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 7.967105546966195, "model_name": "gemini-3.1-pro-preview", "output_tokens": 792, "prediction": { "choice": "D" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:21:00.691404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d5ee063-a3e6-478c-b993-feebf46c3025", "result_sha256": "feb0293e7c6aeb7e27b25a39afabf40db87b1def4d78c10a27e4088f46163c3d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007846, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 6.1909040089230984, "model_name": "gemini-3.1-pro-preview", "output_tokens": 460, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:08.131298+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "019c1509-c422-4e64-b5c6-471eeee033ac", "result_sha256": "f77e898397f9132abc913a1f1fca0e4e882a6a5ad59ebb696f9b6169e8285c14", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.009106, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 10.01918929698877, "model_name": "gemini-3.1-pro-preview", "output_tokens": 565, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:27:38.065625+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "80fa4836-a17b-4a31-b58b-99930969e032", "result_sha256": "009c0286316f865c99518971e513e5d8ffcbc019103bcc23367ce86d09823700", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005398, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.416903281933628, "model_name": "gemini-3.1-pro-preview", "output_tokens": 256, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:23:09.434326+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b388571b-b8e6-4591-981c-56d6616f9f4c", "result_sha256": "2aed2f612a78f4baa422d42e64829911ad1573117f1344d8972b4bc37221b47f", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00685, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.657066929969005, "model_name": "gemini-3.1-pro-preview", "output_tokens": 377, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:14:11.710001+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "eda4311c-44ac-4d46-8084-c65e1cb4149b", "result_sha256": "8ce99c9983dea6cb85e111eead9f217a7483dd510cea9fc59d049e4b26bb5ec5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011518, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 7.086901940056123, "model_name": "gemini-3.1-pro-preview", "output_tokens": 766, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:44.736413+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "88d2dc6d-80d2-48c1-b660-b64827a54683", "result_sha256": "196e607e2b69e8dabf66b9c3a5e6be2cc80054d9902a6574566bdcfeecb22ec5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011998, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 8.154685134999454, "model_name": "gemini-3.1-pro-preview", "output_tokens": 806, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:05.625863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "056ad784-8d9e-47d7-9ddf-183032094418", "result_sha256": "6cbff10d8fe7d0c93131c1d04c4d85f3deca74bb52e7053ccc357b84632368ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00541, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.3664915029658005, "model_name": "gemini-3.1-pro-preview", "output_tokens": 265, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:55.901617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "df624e41-894c-4f64-91c0-d70d31b7a61b", "result_sha256": "bc625a3de4b547c334df18318bd3a5404a36410400d2770266b56298bb783f66", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005578, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.195109204039909, "model_name": "gemini-3.1-pro-preview", "output_tokens": 279, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:33.775808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "12c29aa8-fc44-499a-b64d-87171672d470", "result_sha256": "abaac5bfbfeeaab7423339837ab556dce97280ee5d0949da54befdabfb5f5243", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003466, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.8439926500432193, "model_name": "gemini-3.1-pro-preview", "output_tokens": 103, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:19.067460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "03b9962f-443b-4322-98ea-cd62b9c077f9", "result_sha256": "c5ed19ab01fc62827d8315a0d8e19ecdb5229bacda9023e783bdfc868c4c4d9e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00349, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 3.068550181109458, "model_name": "gemini-3.1-pro-preview", "output_tokens": 105, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:29.529386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f7cca62a-e99f-4ce1-8a88-714bdcbcb8fe", "result_sha256": "4af14185db5f4e41f0f3d37b26719803bb00088049dd2447a99842e0c1720be2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005158, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.238247242989019, "model_name": "gemini-3.1-pro-preview", "output_tokens": 244, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:29.294365+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "5eb019a5-8dab-4771-a0a9-2b7adfd49732", "result_sha256": "cce9828dc17d341cb3050512261be51f63bed17c7a9ad802412fdadbb6660c04", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00439, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 3.7589501620968804, "model_name": "gemini-3.1-pro-preview", "output_tokens": 180, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:36.165347+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "61df100f-f10f-477b-a4b3-0691d8d2b34c", "result_sha256": "d72d74d433ee6f0eef7cc7cd29b4c831faa9081853cca31cae78d644e2d2c229", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003598, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.066824064939283, "model_name": "gemini-3.1-pro-preview", "output_tokens": 114, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:11.467689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a4a171f6-9c04-4574-b7a1-b6a5a3d4c4a6", "result_sha256": "e4b34726ef25f0a1a1ab00ecfc2879ebeea6a25b6fe78d71b41d5b774c52f480", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003826, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.900731120025739, "model_name": "gemini-3.1-pro-preview", "output_tokens": 133, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:49.806867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "27616aae-fa1a-4eda-b015-4d62e6755c68", "result_sha256": "cfe7fc68c240b24005b1c5675b2942575542cb4a595ed3973c29663ad895f547", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00535, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.633882159017958, "model_name": "gemini-3.1-pro-preview", "output_tokens": 260, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:03.923852+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "9b5ed20d-0d94-4853-a9ee-45ffc7f78ed3", "result_sha256": "0328c705d197d0f8ae5439de0c90792216c6940d4cd9b6a43da22211e8418dcc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005818, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.456593708018772, "model_name": "gemini-3.1-pro-preview", "output_tokens": 299, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:13.414735+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "410b37d7-ad67-45a7-a8ea-b4399742161c", "result_sha256": "279371f78a864555663b360fe18625a5ddc9210dc6c9041048e696a3dc2deb94", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00523, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.36672475899104, "model_name": "gemini-3.1-pro-preview", "output_tokens": 250, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:49.850538+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "06365aef-d327-4342-a013-291c84d1aa10", "result_sha256": "aac6dbae67214ed617c7c1e88bb485f4bdaccb28d1b8143c237aff503e93aae6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005578, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 4.056100015994161, "model_name": "gemini-3.1-pro-preview", "output_tokens": 279, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:23:01.278536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2fc9e241-0e8f-477d-a648-9ea139cefa05", "result_sha256": "a9e1e6ea746284ac4a7019fd22fa7f96ff49217d5bacc509c76560fee748d177", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003396, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.3088384659495205, "model_name": "gemini-3.1-pro-preview", "output_tokens": 94, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:50.914780+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b2b562ec-3638-43d1-b193-3fa1ebebdac1", "result_sha256": "ccd13a26aac78903f35017bf6fcb484c005f6520791ef2926bd6364fd2a43df8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003564, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.1591077360790223, "model_name": "gemini-3.1-pro-preview", "output_tokens": 108, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:16.318765+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "30fd60bd-20f2-4f0a-86a3-0400da23a222", "result_sha256": "b31401ce2879d0d82fb66d8c0b634ab5c1318cd076f0b137ad0a835eadd0721b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.6962999389506876, "model_name": "gemini-3.1-pro-preview", "output_tokens": 236, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:07.160553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "636da606-1520-4ff6-be18-0a380009b926", "result_sha256": "9e5a26310840b7e8cef5bc22b430b059e7190f441589d2c1db0dd4d833a6ff8a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004764, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 4.101847501005977, "model_name": "gemini-3.1-pro-preview", "output_tokens": 208, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:11.541127+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "85f2f064-ecd0-4283-aaab-4e0cbd232be0", "result_sha256": "912aa493782c8ef5cb10b4415839c25d8a72ccc9b15938f16c3b5bd67d03a10a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003396, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.4112681599799544, "model_name": "gemini-3.1-pro-preview", "output_tokens": 94, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:23.052318+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c0f38539-b122-4615-bf65-35e72d45be12", "result_sha256": "d05ec68ab21eb8722a865325097270570090598321c6c316f4c54b14fc5b4593", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004176, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.349651094060391, "model_name": "gemini-3.1-pro-preview", "output_tokens": 159, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:15.151582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "cba5f2d7-fcc7-499d-a7e6-2d246f584e48", "result_sha256": "24d4b0f37f3f10389f8e02995ab0197387817c95bca478d6acffbb1830cd58ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003348, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.514783123973757, "model_name": "gemini-3.1-pro-preview", "output_tokens": 90, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:43.405397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "22eb4a4b-72dc-472e-8f5c-6956087ec958", "result_sha256": "6e791ddea73fd08ff43406e5e7d4d3f44575e8eb8537850e3580aed777b8490b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004068, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.7862808789359406, "model_name": "gemini-3.1-pro-preview", "output_tokens": 150, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:52.897507+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c34f2e33-d6cc-4f08-b321-bd2744470efe", "result_sha256": "0104b29c86567b87315b51ad83d39beddfa2f4c3ba89fa7b6a6cb20fa482e6d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003576, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.8603506629588082, "model_name": "gemini-3.1-pro-preview", "output_tokens": 109, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:27.428654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a212f575-3c44-4f8b-b4fb-bf21fd6cc03d", "result_sha256": "428f6dacc8392cf50fc7132fee6570519decef96b7eaa84dcaf9868f82f3018d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00408, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 4.949552930891514, "model_name": "gemini-3.1-pro-preview", "output_tokens": 151, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:30.966500+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3733e0d-554c-4cfd-9451-6605de7fec1a", "result_sha256": "bd33425234c3d0ed96e7fb977cd706b464f170842be5a9766162262e43761d5c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003696, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.0935737270629033, "model_name": "gemini-3.1-pro-preview", "output_tokens": 119, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:53.436727+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "19c0be71-1aaa-4483-bc0d-1efd938afb52", "result_sha256": "1e5a8385bbfcdb24a6c6972f58ed73599bf36c75438b727361b951d4ae43684d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003372, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.8001735650468618, "model_name": "gemini-3.1-pro-preview", "output_tokens": 92, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:38.630834+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "83638b6d-cc4f-49aa-aaac-fb33d4dd1262", "result_sha256": "84ad4b6be30a668a9a45c80282fe44e3e3270c028f5ef07810fbf1fe0d4008ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003204, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.273903712979518, "model_name": "gemini-3.1-pro-preview", "output_tokens": 78, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:11.249072+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "79477f97-6ab3-42c4-a057-01eb3a32a733", "result_sha256": "836a83eb4acd4808769b418d4f7a201da7efe639b43752c0d2f2dddc0e5cb39a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003384, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.3626575289526954, "model_name": "gemini-3.1-pro-preview", "output_tokens": 93, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:37.972602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a84697ba-fbcf-45bd-b5fd-37cdf175f6d4", "result_sha256": "7428c8020769bc02f00bc9eb120081adf7031d99b70e29442e65d622bf1c7a70", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003408, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.935837685945444, "model_name": "gemini-3.1-pro-preview", "output_tokens": 95, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:09.707056+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2a559cec-44a0-44e8-a9cf-933cf9865ef5", "result_sha256": "ebcd7072f07774c12c7642d63c0bc4e11400f9b6cdfcc071f645851a2492168b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00504, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.2081567409913987, "model_name": "gemini-3.1-pro-preview", "output_tokens": 231, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:20.473535+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "947aad97-56d7-4626-bb95-c0172d21cf4f", "result_sha256": "c35bede277796e46b02db7f89b142719fdfd65f98ab22618b5c842664631fdf2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003288, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.317415529047139, "model_name": "gemini-3.1-pro-preview", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:20.228602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f45ea816-fb20-4d69-ba17-8fc366e22345", "result_sha256": "f6953997c81611499cb8088f9777091613e24108f7b490447fcf591a82af352d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003456, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.6787800930906087, "model_name": "gemini-3.1-pro-preview", "output_tokens": 99, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:46.303867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c2264508-2201-4b2c-bf72-9def3729c03d", "result_sha256": "ea0267855ff4fb310d7f98c7bc198e9ba07bdd89909b827b5d90d9b820f4452a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003516, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.76162571599707, "model_name": "gemini-3.1-pro-preview", "output_tokens": 104, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:58.422495+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f7625781-c2d2-46c7-8284-6cbe714d1283", "result_sha256": "fc2023785a61033e88b39fd99802242d7559d492b1a4bd1b2a27d4841616d686", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003168, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.292831468046643, "model_name": "gemini-3.1-pro-preview", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:17.021796+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "5800b1fd-540e-4fb5-819a-78ed8515d4ec", "result_sha256": "b59ce1a66257b1aee66d0c4f9383c0612ffc95ec32c1779e9aef9c34ee376c7e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003516, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.3816705220378935, "model_name": "gemini-3.1-pro-preview", "output_tokens": 104, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:23.633172+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a55ca53-3caa-4eaa-bfc6-244d6e87b2c1", "result_sha256": "07568fd022b1cb6f5f1b4df73c4f699052f21ad0c5317ac420d96441965ca46f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005052, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.4719762828899547, "model_name": "gemini-3.1-pro-preview", "output_tokens": 232, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:39.197426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "8419ff97-6ce5-4e31-880b-830cb1eb773c", "result_sha256": "f683edf23345415549c3a2e9fd21956aafc6eef4d67a61112e5db114c6a0a9dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003636, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.5322925719665363, "model_name": "gemini-3.1-pro-preview", "output_tokens": 114, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:27.614599+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "1aee0de5-859f-403d-befb-928336351d6f", "result_sha256": "d3303df800f9fda008aad3c183101d1ea2b7ac9606658b723331a371a4ff084c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003396, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 3.673526880913414, "model_name": "gemini-3.1-pro-preview", "output_tokens": 94, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:26.846777+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "fbf8b8a8-1091-4918-88df-76394c3e0a8f", "result_sha256": "f77cd3940e30db6c6942065e04b83aa736d2f426e6258d571302212bf5f8d4cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006226, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.343043891945854, "model_name": "gemini-3.1-pro-preview", "output_tokens": 331, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:39.803004+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3bfa441-d055-4af8-897b-e618339e98e7", "result_sha256": "89bf9342f8c39c4fb5514b3505e428d22e74bcecf3f55be3b48ef085d64c81c0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004618, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 3.3396139569813386, "model_name": "gemini-3.1-pro-preview", "output_tokens": 197, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:44.447657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "27be6c5c-b17e-4f9c-b5e2-ac14242a5f4d", "result_sha256": "35a13a8f9aa3e683406496430b95154260dd977c60f7ccd9fe698087efb045f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006586, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.543350588995963, "model_name": "gemini-3.1-pro-preview", "output_tokens": 361, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:04.905402+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8777371-949d-4074-9fb8-5ed86e7bb7bc", "result_sha256": "c799a5b834cae6a8f93c81d072711df56a934015fd2f91e5c8936619fe14a091", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005278, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 3.7549699100200087, "model_name": "gemini-3.1-pro-preview", "output_tokens": 252, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:06.425877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "354d47e7-d49a-4b38-8eca-0562e09c3896", "result_sha256": "6b542607deb166329533802f9ef8b0174dbd39125d5060d0dfab6594bbe5c7be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00463, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 3.8003926440142095, "model_name": "gemini-3.1-pro-preview", "output_tokens": 198, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:17.535590+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "520223fd-4ed9-43aa-b0a7-51be296d7aa8", "result_sha256": "8c4e2aee95bac596dcf296d21e1a6078026f90d257b7b35588e9507d698090d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004834, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 3.449896131060086, "model_name": "gemini-3.1-pro-preview", "output_tokens": 215, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:23.627600+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d114401-0e4a-4a87-b2f1-06df7d28ee13", "result_sha256": "91d446c8514df26fe7648ee1498632230ae4e973f06d5f780efedd9d6ea76a52", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006862, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 5.055059200967662, "model_name": "gemini-3.1-pro-preview", "output_tokens": 384, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:06.831686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3532934-6e28-4728-afdf-b6ae0c591943", "result_sha256": "1a7d922e86064f7151310be1008126319f550086acaa4db6f3d6dd943843e785", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00535, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.392102210898884, "model_name": "gemini-3.1-pro-preview", "output_tokens": 258, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:21.105375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "dfed54cd-0899-40f7-ad88-ed1c3aa1d4c4", "result_sha256": "f539408d041e80875188f32c78423965dfe7be900af77e5bea90759459bef8ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006586, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.257238844991662, "model_name": "gemini-3.1-pro-preview", "output_tokens": 361, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:02.084160+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "2bba5971-eeb0-4119-8e3c-a4796e1ae308", "result_sha256": "7bb399fdbbaf5327543a2893398e1d292a87e06132f64c84e8f8c54ea7261363", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007126, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 5.094746188959107, "model_name": "gemini-3.1-pro-preview", "output_tokens": 406, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:39.146747+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "70995042-ef88-4760-8744-e58af5b64360", "result_sha256": "ba33ee6087890e956facc2970ab01326af60acb7f5b829d45b073d7b2a5e0214", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.643324153032154, "model_name": "gemini-3.1-pro-preview", "output_tokens": 395, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:47.467557+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "7adc913d-5ef3-4215-b500-1cb04b62a155", "result_sha256": "c47e3629ffc4bb0d368d0233940e499726603a432189ba3801e2aab94a415296", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00613, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 4.4835802969755605, "model_name": "gemini-3.1-pro-preview", "output_tokens": 323, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:18.682493+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "60e39d66-8d22-4560-8aee-50e5ea5d48f4", "result_sha256": "762bb56b5c951b676593e969704bcb56807eb01b4861fd3d88f53289a3158023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006088, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.422139151021838, "model_name": "gemini-3.1-pro-preview", "output_tokens": 318, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:01.601218+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "693af285-89e2-42b5-84b5-562addb6a8d5", "result_sha256": "ddea320571b15e2b578d647ada669547bf70385899da4c832c221502a06242f1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006292, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.239256981993094, "model_name": "gemini-3.1-pro-preview", "output_tokens": 335, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:36.316975+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "35e46449-685d-419e-b9c7-67957bce3bac", "result_sha256": "fe4a829fd9d464bdca15fcf8d5952e2eda5a18d47ff86dc4c91813386cd6c71f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006388, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.525633545010351, "model_name": "gemini-3.1-pro-preview", "output_tokens": 343, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:02.680295+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "08b3e332-4ee0-4424-9595-92b1475ab627", "result_sha256": "570c301ad142ea37e0065769073efbb90185334fea50c6ce21d6b0acc4b19d30", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006412, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 5.034085288993083, "model_name": "gemini-3.1-pro-preview", "output_tokens": 345, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:39.190043+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "cebedca6-15c0-4fae-b67f-0afcb1835533", "result_sha256": "b9c03f34b4ec333b8096fad6e408462e395f94a5805260a292679b0ed5156f90", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006304, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.500501325004734, "model_name": "gemini-3.1-pro-preview", "output_tokens": 336, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:48.094698+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "de517669-3e34-445e-b14e-7c4ad6ff6190", "result_sha256": "a6d9e59fb038bb3bb3b0fc77f22139dc4cd7920d8ca2622bc3b4f6f7f66f6e65", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01282, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 8.309079683967866, "model_name": "gemini-3.1-pro-preview", "output_tokens": 879, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:47.881934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "fdf876a1-aeaf-47e4-88da-75e7dc22f75b", "result_sha256": "c41d17cd2c8ee375677fc802d855450e05d7b020e1384631187f9e4bdeff06be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006532, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 5.149035249953158, "model_name": "gemini-3.1-pro-preview", "output_tokens": 355, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:56.990706+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "73ea52fa-0422-42db-8ffa-e3cf78d33bda", "result_sha256": "42047470c5081ba3e7c818d0c520ad302e16c4a4eb5dc9fb00df93eb08abf65a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005692, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 3.9606221690773964, "model_name": "gemini-3.1-pro-preview", "output_tokens": 285, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:38.742316+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "858bb771-0367-4af2-ad66-d468a2bf63c8", "result_sha256": "2ff8a8a7dce487267431e34fb3b24d0fea61051d4bf77b086cdd0cdf81e4965f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006616, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 5.460470407037064, "model_name": "gemini-3.1-pro-preview", "output_tokens": 362, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:12.782892+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "29fdc131-9c67-4bee-953e-0971a3fcdceb", "result_sha256": "283824403b6ebc63bfc33bebd7fc7c858fa2820aee39876bd22a3e978231ddd3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006412, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.614468805957586, "model_name": "gemini-3.1-pro-preview", "output_tokens": 345, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "google", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:59.236085+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "cb2b8ec3-1ef9-4d2d-aee6-0fcffdb8919e", "result_sha256": "843fae0f30a073a1aeafe0a78228b19bce9d570f4696d2011e72612f3dc4eff9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006616, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.772019983967766, "model_name": "gemini-3.1-pro-preview", "output_tokens": 362, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:46.355260+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1ec0f53-463e-4024-8235-7274113487f6", "result_sha256": "4f1a670a341474e0f8240277a6f41ca1baca5c651b7a9cc2370ad04f0a687f02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00706, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 4.95698898693081, "model_name": "gemini-3.1-pro-preview", "output_tokens": 399, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:55.696618+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "72186336-0a7f-49ce-90e8-eca73b3ec003", "result_sha256": "c5ffe8840a7cb3e9df335fe1d37dc5c0e14026447e1526ed8cbb329116d2c3b1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00344, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.0959997860481963, "model_name": "gemini-3.1-pro-preview", "output_tokens": 101, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:08.435495+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "f68ca7e3-be48-41d4-a1c8-9b10df0008d3", "result_sha256": "bcfe6bd1f574f01c5908faaae79bbbd206dd551b14099562a5a91d5e81b16bf7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003668, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.486550014000386, "model_name": "gemini-3.1-pro-preview", "output_tokens": 120, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:34.727703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "0721568b-255e-4aa0-a7cf-9419d18befa4", "result_sha256": "2adce8b107200cf554583aea4e839fb732c15be33fdc64985bd65b1303cf730b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005384, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.5971743060508743, "model_name": "gemini-3.1-pro-preview", "output_tokens": 263, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:53.024896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "5c76120c-3427-40d0-b842-a009958a81c1", "result_sha256": "78ee4a6cbad9639577fe84ed67625aaaef5f9e7bfc813d81a67efa8be13bbf03", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005852, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 5.428437864989974, "model_name": "gemini-3.1-pro-preview", "output_tokens": 302, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:15.023230+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "a37ffeae-8cbf-4775-a274-656ecad6d7cb", "result_sha256": "b797807a6486603fc12c55373a7ab9bc688e180ff6048348faa0d5ac99821609", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007328, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 5.328509305021726, "model_name": "gemini-3.1-pro-preview", "output_tokens": 425, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:08.004235+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "704a109f-6e53-457f-9c45-52fc19c90d96", "result_sha256": "e3c69f231628c24355f8c6b59622ab8c7fc3c9a8c45da9d87c23875e87f85c8e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00518, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.9562194580212235, "model_name": "gemini-3.1-pro-preview", "output_tokens": 246, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:54.042238+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "093f17ae-99d4-4482-9e8a-35028c696a34", "result_sha256": "a17ddba7f89ffb24db64b070a4ce4a6ea20e60acd893fa312bb50fd1a7ce1f53", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005396, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.892907712026499, "model_name": "gemini-3.1-pro-preview", "output_tokens": 264, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:21.389579+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "1dc60029-975b-41c6-ace5-b4ec1bf2291a", "result_sha256": "214db5d5b231c5a5a29e12553cb0aff77a31e772e9954a73113be22ae0fe7ba8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00608, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 4.734445938025601, "model_name": "gemini-3.1-pro-preview", "output_tokens": 321, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:10.215014+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "26ab3eac-d047-4d7c-88d9-4f072df6df54", "result_sha256": "5e27e32167a39acd6b19a26c0cb5f9d9b0ad684f46cc82006955414d4335e87a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005528, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.986104427021928, "model_name": "gemini-3.1-pro-preview", "output_tokens": 275, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:52.638246+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "294fbf44-513b-49d0-8702-6099413cd883", "result_sha256": "47bd197a229c6c06f9799f01648220752076d37c6b847e10f13934ae61a23008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00368, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 2.917451735935174, "model_name": "gemini-3.1-pro-preview", "output_tokens": 121, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:32.962897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "dfb98fb4-b517-41d9-8f6d-cee4681124ec", "result_sha256": "e06536286720721fd919c63495dc984bad00c005e251cf79e68e42cff870e845", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00554, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 4.193156302906573, "model_name": "gemini-3.1-pro-preview", "output_tokens": 276, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:58.485327+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.1-pro-preview", "status": "final", "included_in_comparison": true, "source_attempt_id": "c613dc10-9cc4-412f-af83-99f443fe159e", "result_sha256": "d1d1523d2e17dd1841d9fa8a22ae25057a1f3e47ce0eb543950eb4730c43de23", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003932, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 3.835171901038848, "model_name": "gemini-3.1-pro-preview", "output_tokens": 142, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:46.476561+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "5fbc939f-d59d-4734-9b85-5b1b465005a3", "result_sha256": "964f3e3a3a24d372e1687fdacb00c872c13e62654c24ab334d496c4e70320b3f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.1459185369312763, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:49.252718+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "2882ca66-1fec-4bca-b2a6-67186e442d7f", "result_sha256": "1ee1a006db730434690a18b980992c47d96b684f017a76d4b47937cfb5bfe5dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.8349175590556115, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:15.940144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a6b7e8b-a598-43be-b7e5-099c2d47b628", "result_sha256": "0af18690b71ea04836644239e2d774b67d47c08cc76cfd44a6244634dc097a91", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.9861023450503126, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:09.051565+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "dfbbc386-447d-4b4d-b825-adfa28f860c2", "result_sha256": "aa2982d0288c81636179ed097b32b10baa65971b52d282f1a94fda755a2c1797", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003582, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.1037840569624677, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:12.947133+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "21e09689-5186-464f-a628-72090b44d64e", "result_sha256": "d55ecc039c5c351b3ec02013aa6d672f045a47ba2a0e480cbc99036c1aac1cf1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.3587010781047866, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:40.021331+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1558742-eb99-4680-91d6-2960184b951a", "result_sha256": "15182eb93a2d4dae2e56d239799349ab38865f4163bf66ad21d545f8fa58d391", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.8469828380038962, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:54.312272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "461f8f57-80fe-46f0-a072-9c43a1485019", "result_sha256": "32d10ba3eebfb1dfb96849181d442bd47465cd427f3351a93d1b599347c90384", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.9523923260858282, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:39.678673+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f48fbf2a-079b-4c41-8e32-fca3768da972", "result_sha256": "e7e2b2f544e926d9d2361aef0404c2a0c3474bdd89ec74ef47920d8f3d30a5db", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003582, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.8306983150541782, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:58.538755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "07054a49-0374-4560-a33d-6303057b0a2a", "result_sha256": "3b404d78c5d968bdadcc5f3cce3034e424757bfaa142357333bb998bf88fb71a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.9634422870585695, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:30.265348+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "40c55fbf-571c-4d47-9b53-5a22ba8d962f", "result_sha256": "b72fcfb16a97ebb2bda3094c2d02ac361b1ecd2b5ccf4f65765db1c6da40289c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 0.9113805490778759, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:52.855222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "d286565e-914a-479a-9493-ec304484d6a0", "result_sha256": "1c0dc7a282974b4923dcedcec8877235e66cc5371dffa1ed6eebaef64b577b8e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.2381884990027174, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:13:30.584512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "423d788a-3a7d-4caa-b1f5-4c526e8ecae3", "result_sha256": "692cd27f5e222e2723ef2e81fdcd776f0b80df77d3cebbe4b0340a71640bfc72", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003707, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.347909044008702, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:52.292982+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6550f0fd-b7a2-432d-9db2-44a1046dbf8b", "result_sha256": "bdc2a2baabf1696669c356315ffd1d5a70191caa6685b6738d9c17cb95b00f00", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000378, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.1231810949975625, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:56.481157+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "2074d84f-2067-4857-bb42-df99a62e27ec", "result_sha256": "2e04669309f077682a7c8924795995bad25d8bdc83ab1251e33548623a13a9c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.0346066299825907, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:00.991639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2435769-2569-4c07-acd1-8e54ae67ae58", "result_sha256": "df26b55cadbb3bce8758ec2b3998256e1e623198f2efd51de0f07b5539de4514", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.7606249729869887, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:00.652926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "97e4f46c-1ce6-4d6c-bd97-38bd1665e51e", "result_sha256": "625fed3af7e9e7c929f13fd4d26f11112df2a7b3c4a75ca867ff5ef073271b02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.9407228549243882, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:19:05.906759+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "efbd43c5-c4a8-4b17-b771-7c1b801559ee", "result_sha256": "addf3906c06244cc9cf5498b84802cc407effc435c411fef40c035296111fcdc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.873080765013583, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:11.790264+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "bccf1c70-c7ec-4cce-afcc-2c1ab56148c9", "result_sha256": "d671e427e161f1edbc4685604d12af5ef4f941305f213352afc7a57498505292", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.2992050889879465, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:22:12.200145+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e3a3240-c774-4cc8-aa18-2377c7fb2a99", "result_sha256": "c7c3fc46a3a332d0ad0fbb065976ba31d59a03af6dd5df0f707e92ca23080207", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.38589277502615, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:18:44.000072+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3316e605-d919-4d6c-bbe7-c41deb83d4bd", "result_sha256": "3743c1af7ce5c6109a93b93c4bac5262b1ceabadd9fdf4be1e8be7738c5e1d5b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.8670966729987413, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:13:42.581668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "1718f6e8-8112-4e13-8acc-5783d0e9c570", "result_sha256": "5fdb7e3f325db7d671ec454941db485fe687eb2037db6d49b2a3c4d3c8b02b61", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.7582473671063781, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:27.267841+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b0bd8b33-68ca-4fea-8c5c-81685c78432f", "result_sha256": "93bf34db1b7c09913c291c911169a97218f272d9ac129200386dd56fb330f86b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.0562504030531272, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:21.870556+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "7210efb1-7a12-4327-8d84-36dd389f374b", "result_sha256": "7796eb6043f242b8fcccfdd380d6074007200ddbdca9149404f93d6b02724dfb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.8524426079820842, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:07.323045+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "02e71302-c34a-467c-9ead-fb7dfdcd472d", "result_sha256": "06a060a64e699bf37886887dcffe61b34c9420d6e5696f10b90dcd0eddd59143", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.9906035379972309, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:33.210778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "48ab25e0-a773-4743-b296-a7446b36454a", "result_sha256": "3339530ceb6455757c31704c2a773be404b32dd1eb647518e4a939ea4af482ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.9013200149638578, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:11:51.067345+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ca86a8ae-447b-4cce-b04a-ee770338697e", "result_sha256": "ff1c451dd2614743e0be90c137d294209eafe18388ce25081637109c8b277781", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.6223355419933796, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:03.614142+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "df6f954a-d675-4dd5-a59d-a38d3b5f637f", "result_sha256": "905a1566962babd8a184d26b8dba619a6991f47e620699cd453de9392e409b2f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 0.7467956970212981, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:14:57.522704+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "74b43cba-514f-40d9-8b61-09a8604abaa7", "result_sha256": "b4697fe91dbbd9878acf7b53c86cdaeab3b9ae63fa428a8013a80d9f0ebe538d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003905, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.4206607500091195, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:18.637770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ce87fa5-9ffd-4815-ac06-e7d7f36501cf", "result_sha256": "05b79677ad2c29f90a31fa9ff75ef5f465a497029c6bcb8e3125237d4afa6097", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0251946720527485, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:39.983479+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "02d95d1f-eacb-425f-8236-e8cebc179733", "result_sha256": "9858a50f98531d4f1d715e2eb53561e2c3c6b1ccf973fc98f398f50e40ec8d81", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.8855716140242293, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:11.498421+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "640af946-af9e-48dc-9ecd-7def52653e3a", "result_sha256": "c45ddc305fe90ac6e296be06c9d94ff60cd255e23a2f9bf2e1018071a31b6a58", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003495, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.8828969240421429, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:29.911796+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a1cd4c79-9c85-4ac9-b0f0-c5f4ac3c4f4d", "result_sha256": "314fdc011f0d80a21b62ec4393f6cdf580d7d4cc3d0c1b443192e93437fb711d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.4561163529288024, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:02.787056+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "08048c98-707e-4543-9851-55bf89cfef8d", "result_sha256": "5497b9a73053c0fcf58a33d6db956fd7e854820f8ff33c23d2dc7330ed58fbff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.40542722993996, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:33.597882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "25d31dee-3b60-4441-ac34-a0437ad3e0ca", "result_sha256": "217d8a3d45e09bec879f6480351f063315f8551fe7fb2c1590366eed1fd1addb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003495, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3747723150299862, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:00.844684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3fa67e96-03d0-4f10-80f5-9bfef3c6f7e2", "result_sha256": "4d0b418838e9a6d6b573f3aaa0b42491785fbe4b46545378c0b7ad64046b1ad1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0523491769563407, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:41.667615+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "df0d1d67-ff36-494d-a62a-eb8c87ab0ee9", "result_sha256": "9d134e6c187c0bd3bf551b54b6a9595d0587aac8408f81b9a4e98f4da794bb74", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.9997370389755815, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:17.209274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a07926c-47e1-429b-b09f-6b1a77bf02f6", "result_sha256": "1ed65fd5b52af6cd2ce8f4d673abdc525ec461a8517a79fc5b7f450188f52724", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8845391189679503, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:45.409003+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "db65b62f-db96-4095-bc02-c32ad0c5ebfc", "result_sha256": "367ce750f850955960097e427dfbf1e935ee4d3181127f26508a5ef8b94eb70c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7594605199992657, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:26:32.943464+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "92abbb60-948c-475b-aa16-5d5caa98e509", "result_sha256": "a239d06e60ad875e9c2162d3a3898cc0d812e5693831853efbb631775f6d34a1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8778869779780507, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:56.633377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "1450038a-16dc-4a83-a5e5-3b50da7fb59b", "result_sha256": "b59750aab8b554ca11c4b5feabecc6791d9771e12840472c6cc99e773c9a718c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7551677309675142, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:56.066064+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "26c50489-63fd-4b65-9a33-4c02d74b0ee2", "result_sha256": "3f2c70ca4d010336d7f85ad6cc3452ec6293098895298330ab76575ca9b674e5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8416627470869571, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:12:16.384099+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "513b86d0-688b-4fd5-b253-002ea7fe6697", "result_sha256": "c202319d8bec53754916dd56ba836ed17fb2f3796eb68986b46d82625ff03e41", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7882176489802077, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:11:33.461956+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "9a52e050-31ec-4084-a0ea-f4638b23e2b8", "result_sha256": "279805abeda149a494b3bbe6a02a79d190d9d026e5dd80c9c4bd288efd10a76c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.1800083990674466, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:25.926932+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "601c90fd-909b-44bd-bd4d-65d3f17e8bcf", "result_sha256": "c9436c6916263504b169e2a7a5fb09fd6d9cd2a80bcead94ac7c06e78770e5a8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.9410715179983526, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:19:16.628590+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "bdaa41a9-dd1e-43ae-b701-f70b1b6ee6ea", "result_sha256": "4855d4b496060983706c5d8418f6a992170f9cd79bb43dfdb1d86042198ccccd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.0388778370106593, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:24.438099+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9c0d0eb-5817-4f7d-b064-8ba88c7efe86", "result_sha256": "bd1948a0d02a907f8784bb7246413090e9ddde3be14a2204b39cfc90eb5311aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8386752229416743, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:15:22.930098+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e57a9189-3862-40e1-aff3-9ed62b8d181c", "result_sha256": "faf69db31048a0cddfbe881db64d579a4cbd23bd6428948ebbba30ee91fc008d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8732684089336544, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:49.806063+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f3c789b4-0376-4b1f-9204-24f697c558e1", "result_sha256": "97c5e36b90c5ce465e325cdbdc3b4f6d5b1276a13cc60e1d398ba536768b8612", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.6875983340432867, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:25:54.256575+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a382a1bb-9da8-45e8-b866-132b127cd634", "result_sha256": "fb7cd1bb557afa44c2fa00f490008e0be488803be1557deb74ab3f6ff30e7b55", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7456864599371329, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:26:49.398720+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "00501d94-256c-4d96-9026-0e2b061ccd98", "result_sha256": "d614959f03dc6499612be1ff77d6472178005266a598c078e01bda8efe6e6abf", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.9344471249496564, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:17:47.393644+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "24307349-25a3-4c10-9a33-043edc2cd2c9", "result_sha256": "9c47ed3ffa802c0abcbe21e2b4e45be1ea38347fe9c627407df4ed5d5c4eb88c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8088879610877484, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:36.228140+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "41eb620c-cb9b-4dea-bb3b-f64556fe6130", "result_sha256": "48c80d308f63c39cf50a3f5f162c5e76fadf8bd323eb58165a071f92e6ecf384", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7968223060015589, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:26:23.749432+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6837a1b2-a41c-49fb-a8ef-ba668caeb284", "result_sha256": "0259e1901611b9422bf898555325c6cf87a745f9a8bffc0b85ada325264fcdcb", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.6595720970071852, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:23:40.556399+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "d31f2863-30e2-4f5d-b549-92e46335b180", "result_sha256": "a589b278913d26d9c36e0e34504e483c4eb2bc02cc7197e86260f405eb0fd415", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8088005949975923, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:25.324371+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b1f2c767-b78c-4ee4-8254-7da1f298de65", "result_sha256": "b146845993e7775c840e692e878738f14d3900db65805735003078b9f6b7bcd3", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.6990466930437833, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:17:02.279990+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd93ea74-24c8-4512-bda1-3881867cd145", "result_sha256": "bdb9526108ce4e18f9f3fbe8a4258e385e5d36b999921a4d3e44449cc6f4b37d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8366094869561493, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "google", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:21:03.749989+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "7d529c07-8030-4f5b-a071-ccb2be77c5ff", "result_sha256": "4015a40bc429c511aa3528d268c967deafe96a286b96121196aa250400085acf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.2986678979359567, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:57.844397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "7cbfc6ee-fdf4-44e6-9f46-a8517057c581", "result_sha256": "19001a0bfe9793e0abd56718e6b99e1e897ad0e137f5053e88e966bb9f293a91", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.8845387200126424, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:09:08.447310+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e7535979-a51a-4098-ab7d-7e3926637d1d", "result_sha256": "8effae0f9c082be333a27336259a25fda2fd2348fd340acf398c6026ed345650", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003789, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7518500150181353, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:34.155582+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "18d0d23b-4658-4988-8f84-f39d113675d9", "result_sha256": "9b9030ba1b42f318ff84962286c98cbd85e2a34cba0507506370a8d129865e21", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00039139999999999997, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 0.7619712670566514, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "E" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:15:54.760132+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "65f99ba8-5f1d-44c1-a8b5-5eec10169a13", "result_sha256": "c6a398480fdc3586f90ea0169ec3a58501b151f2b9299328698cef3577438dcd", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.6965326819336042, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:09.760823+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "80c0c521-d8c0-4160-be81-5fbbb875ba58", "result_sha256": "9c76c7d05a0365e89ebde6241087ff2b2b2b0cc3132399a7decffbc2230cf5b0", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.1911816929932684, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:53.353804+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "c34b21af-ee8e-4903-9c87-69e23369605f", "result_sha256": "0bf7368fd7c268b0144c0524ffa19e7cbc93f257a8520cee9b932e74e11b9bf0", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.9511572680203244, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:27.932516+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6a2a5ac7-eb80-45c2-b2d1-44a1fdb67782", "result_sha256": "41e48703e02fb12f9da118e278ba9d9948891fa3a83e072435f9722fe9af859d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.1127838440006599, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:53.465807+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ba9d60a2-a0e2-4246-85a6-8b522d00f6cd", "result_sha256": "56e7bc05f684bd4fe32bc2d9b6891b85ad2e0a257895c4f20f1fe80a5e1c3dc5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.8089474890148267, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:00.353036+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "38994839-f3d2-4fc4-a1d6-06488a21bba5", "result_sha256": "a74a55dc6885b015160c1799f49cecf08c22b814b26c628b340be698543b91d5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.804426756920293, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:00.806187+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "bb2389aa-44da-463c-8e7e-ee9178dbdef8", "result_sha256": "8ddacf78c20e700cf30a7ea877d65e5307732e5b7f6df405a267ef9945f85565", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.8176993959350511, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:18.955193+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "85be7a31-b3ae-4048-8d35-945ba2367c26", "result_sha256": "eb01779cc65c75b9eedc78783be332dbfb2a606a5182095d3bebb0d3b63ec333", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.4478617920540273, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:36.432102+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "eab0dad7-7220-4672-b521-14d2af59d0c9", "result_sha256": "99573e1e3c6cea6a9f40411a8e3ac3c55643cf6a5d3b6758b8dfd79664ef3ba2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.1412842309800908, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:07.688859+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a8959fca-ca9c-4740-82d4-fa1e3b0d7d6e", "result_sha256": "17a389a62ebde0424b7099d85cab0d97eeb5672df1307c5bf6126013d77fd6c6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000368, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 0.8453296589432284, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:34.224855+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c67cd5a-be9c-4bc9-b2f8-44e3ff7724f1", "result_sha256": "b1853552774176476ea16462995f90f1a78a7db2c68c1a2fe367357fa6085adf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.031755618052557, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:18.950528+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a678f07b-775a-4bc1-a7d5-72e30024cd95", "result_sha256": "76bc4f2d6990c4b79cd6b6ffdff3753f48284c44a1d166c690e2224f226e7349", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.2464488060213625, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:58.452162+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "fd3c5cec-161b-47d9-951c-1dd5bc81970e", "result_sha256": "69a2cff1a5f1e843e5701b113617277d011664cee3fa751b7967cc849bcf5a36", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.3854733059415594, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:09.965128+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "95d4454a-af6a-46d5-8121-c5f1bf871ecd", "result_sha256": "939be05bf381529e5b8378f5fac67059da01cc9fa73b64ef9c20d359eb8e15ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7347398030105978, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:24.207855+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c839085-7836-4448-b040-e549558b5a92", "result_sha256": "6fc21e5316f6e91edcb3d826b2c8555605f0c9c2db61cae4082ff3be47f3aa4e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7265717280097306, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:10.597445+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "188a0b50-996b-4b94-b0cd-9f3fbc17047c", "result_sha256": "8702f0f97cffa1ac4b874e1443ba79c8582a6f2d75a8094726dfbe7db5ee4c6e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.8028984219999984, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:42.615114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "9b4685dd-3472-444a-bdab-bcc0a54419dc", "result_sha256": "eaa3aaaac6100033ceb793a02a350f3d8e0db890b28689610e99b5c8f84c6049", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.761692548985593, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:47.439626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6993ba16-ccdb-443d-85c1-877dac07e6c4", "result_sha256": "1ab4c60dd598b2e5f371265dea199bec07105350e666641640ef9968f9e8eb8e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7085910870227963, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:14.710793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "5ed9158c-0940-42a2-87ea-e931fc562797", "result_sha256": "5b698a123c082d9232218a9ff7d0214eaff230e9f94ef0bcea8dd84d3ec5f839", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.323662888025865, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:08:55.680068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc135f69-7a3f-4b43-be61-5ebceefb2e57", "result_sha256": "2076bcfaf4834680e782b9128e581ee91ed8684815a7492c3e02b7d4b524605b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.9890992300352082, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:59.718973+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6e64f08b-0ed0-4616-ba2a-cb6bf91ca34b", "result_sha256": "f9665289877573b788c2b9bdda504d872de2514a0c56b91530419d405283a59f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.0481992569984868, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:13:38.556517+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f4549309-921a-400e-85a9-d75f9b686983", "result_sha256": "374cc3bb2a936b1ed8aae8da535782d2aca910caf38dd1af1a0818c48409baf9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7898848729673773, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:29.593350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "50066a40-2c1f-4184-86b9-6d6df7bf35f7", "result_sha256": "c0a144105d84ca5bc58759d59a17f41d616ea0ec26112c32748ace9cb1b314d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.821519845048897, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:33.617517+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e10357dd-6f64-4d5e-b0b1-632381af9489", "result_sha256": "972fe16ad616f249b2e76bf6adec82c28da01578020355b3dd07e9f92a20f9d4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7675179750658572, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:21.633739+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "980e149b-de2e-46c6-8c1e-ed685e6bbc96", "result_sha256": "306fb427177dcbe0432a1ee2c4ea2481f83a2f2e953f9a7e2351b989f80b2699", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.9614038040162995, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:04.114441+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "2caa2814-d4b4-46bc-a1e0-94bc8064bae2", "result_sha256": "144f9365b489153249b82df043d804cd15bb23b7d552e8bd9207328b98297c5b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 1.003940571914427, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:30.897442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8fa6a3ea-664a-4200-8214-fcb8dc128670", "result_sha256": "184cb68b7318b6387b1beabd0fe17b1f67870ee311dd639a0aa0c36e6bb71ee4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.8763752470258623, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:12.718001+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "74a3225c-f6e6-4daf-9c3a-948956ef79bd", "result_sha256": "77d508b0b047e4691f31710090a05d9ab1ecdfcd61db8a4ad56ae76cdec2d6e5", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 1.028691554092802, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:12:29.599385+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "d211d298-0726-495d-8f36-8161feb0f839", "result_sha256": "965deb634b4e0c091792041047d276bf92b93415ce106741d4714f10329c75fc", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.83835165691562, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:29.131850+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f4d2c833-3fdc-4db0-b8e5-71db0ef18012", "result_sha256": "d7ffccaa4b4439cc21697cc81aa7c3feba0c5336404eff3e12efb17d815c9b40", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.8625750930514187, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:39.566562+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ccd17b08-df4a-4934-bd8c-d5bb53d0b2df", "result_sha256": "137cf2e0e7d0111fdeb038d1425d4c8df7a9f71f5c16c1697987049ac74d1c46", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003543, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.7992349660489708, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:58.886202+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "2a45e232-a74f-4e16-b0c6-ad0cfb4a0384", "result_sha256": "9cbefea1525a07aebf73cc54c307fd572a2a976f92bcc81845a2afdab2bb5ea6", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003543, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.8720822840696201, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:25.575542+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "57243f8e-8cbf-4613-a2e6-b7bb3c50575d", "result_sha256": "87cf9530aa9c51a42e0a9a874f862bcab6a6d9cfadab0fd4081cab5b4db12a6a", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.7960829819785431, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:39.141915+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "9a99d589-fa6f-4547-9c52-dca5e8204894", "result_sha256": "b479b5668dd1ceacb1603ca03969efbea9fe18175354c7581a5a9a7c68e94c32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003543, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 0.7549052060348913, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:16.556966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ec967940-f380-4a09-8522-ee5aee8e125d", "result_sha256": "7f5fb5dfbc672ff27c2a3a279ce508865c04dfbfdea305417389c6fef5b9a357", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 1.0106478090165183, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:09.320305+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "34d68988-2ba8-4701-ad9e-158b85cdc9f7", "result_sha256": "4160a6ef776339a8ea3b587bb266c9913d15ddfb376ec77046e33a6051b7d0f4", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 1.1096744410460815, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:21.515192+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e28f254a-c52b-4abb-9bd0-a188ab4f385b", "result_sha256": "3090f233abf15a4323190f83700138333ee925748ca7b1625ff6ae2bf001c164", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.9211576940724626, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:05.096153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "13fcad46-42db-4d89-a8ad-ff29f9a4476a", "result_sha256": "0ae5dc7e91ac8ba3d9442e8495e59a541b3d714ea38c8328b4350b016502e55c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.0821924679912627, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:12.105603+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b67cf89a-6ed3-4f26-a92c-218f0927345e", "result_sha256": "f12118d682af5e3695ccc822343e35a1d834d746fc72de27a09112a6352136fd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.8196095380699262, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:02.535278+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "278b425c-664b-415f-93ca-23e76651f6a3", "result_sha256": "067681c30068ded4f629d616560a470a41e605626e30c83b2cb0e75ac42e653f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.9243453930830583, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:13:53.048120+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "37a05cd8-4fc2-42e1-ae9b-24a2d1d1efee", "result_sha256": "f67bf64c30a5f83ed28fcb2f303f442fa9d71f48fed8cb61d82e8e82efc0d3c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.7804537810152397, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:21:31.988935+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "46621f9c-95d8-449a-8533-612f57332967", "result_sha256": "bcde2b599e30df2f641aff8432ad181f06af73e924a4f94683c57ee4116309df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.8065449090208858, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:44.039007+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "352ded98-d04a-42c0-bb89-ae9ca2a09802", "result_sha256": "381f4d4d32ab8d2f71520e5829094fdea6f4fa3c82c9d7a8c09ed1027b87aa75", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.0073577360017225, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:08:51.772730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "947d894c-4c44-48cc-8751-130942d16307", "result_sha256": "af98feea2abf484957308bb4da938bb9ec93f92798d73eb5d987ef6dd2a74e12", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.8413208499550819, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:04.484503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "fb5788d9-936c-4a6f-848f-047b5075b5a6", "result_sha256": "3e502467c8daf406c56d6eb69c9570a05a8302476a30b75839e42332d8d131e0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.8971667269943282, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:13.400105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc3c3997-6237-480f-9d06-17d4d231225b", "result_sha256": "0879f59ad934972a96f3514f2011e995118e5e43eadb1abac44e4c50d50d59e9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035009999999999994, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.2616434829542413, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:27.172369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "704d5592-52d8-4230-8c6c-1dbb808d70b8", "result_sha256": "36660c61c0200121e484e965dbf4f7755039929e7a7db54e8b17d20fa71f975e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.9915095849428326, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:52.970116+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e504199d-518a-44fb-a7a9-cb59d2c3a4c2", "result_sha256": "734e5802950cbd08ee0f96f3ef4d5099cd050c9bbda36a3d58b551fbd5b3705c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003626, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 0.764431606978178, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:51.521474+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6f32498f-c8eb-4fde-b6a9-4789b3f62957", "result_sha256": "85b12ae4f314e40c4acfcfc750b98a3139274c59dc34d87ef6cb70ed7a3ce650", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.8650625739246607, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:54.749351+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "455fad39-d679-4213-8a44-4ef327d13621", "result_sha256": "dafd97de1228d1079a35fa575f7a5c4bdfba975c49449a99845900904cd70968", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.7777792930137366, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:19:09.328361+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "c735caf6-c798-4705-93e4-f30427e22355", "result_sha256": "0700cf809a06ce3c10c520bbd870ccf4027a12c1369de23ec24671a405105817", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.9630382580216974, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:08:43.769746+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ef6b399b-fee4-4099-b52c-2066442b741c", "result_sha256": "d3549b3868f8b48576fe8705120330e91a98476470f6b9294bef04fcf142dc34", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.8249003910459578, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:04.147322+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a907d2ed-3865-4649-8976-c259f4498174", "result_sha256": "f60c9d82cdc689e71c551e0af133dccca0b52a4956522af97594eab695fc9c06", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.8878619209863245, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:26.209793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "2272baea-9fda-4b81-a13e-cec2cb4d0a82", "result_sha256": "07c6265dde10e1ad5d4122e622ce192b0d20b4bff67e9cf4362438186e662c32", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.066970085958019, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:19.456002+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f90146d-9b70-48a6-b875-db9f5ae9d4a9", "result_sha256": "ab8985820493d21dab4c7ead75ca8f98ff693e5602afd3d7da0738e7c8c3d767", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.0586135110352188, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:36.268739+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0695a92-f95b-4cf2-ae43-991e465022cf", "result_sha256": "7dcd6987f0c2610688c83a562081c9a18b7a3ad6bafed1e88ef4d3fb4d1b90ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.9971439059590921, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:40.635774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "2ee970cb-98a9-458f-b492-4a7b629bf301", "result_sha256": "6acde185c144192767a8a58cd2b7cd41e19e303b69e31b80794c60dcbc5feac7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.508151096990332, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:58.059051+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce1db3b1-7964-4e39-93c5-bbab06207250", "result_sha256": "579c7d6051cb46c367c35e1154aebec973f3ac4770cd13e5e8a2aaa7098de7c3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.049276147969067, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:05.897545+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "54f6b423-9353-4c3a-a47e-7f03ea0fd4f1", "result_sha256": "29799a7393cdb6ed153aee62c1d3f13138c541cedbc9072ec55d2f80d1ae813d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 0.818640039069578, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:15:28.012953+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "7480a2dc-a31f-487a-9482-6150c6d69ba4", "result_sha256": "76ab626df0bd05508b259dcfd4502ec17b2f4a7c72cd687ace2ae43d7bf40ffc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003695, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.1666187830269337, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:17.388949+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3d2cd3a8-f362-42de-9e0c-6920bb90fff5", "result_sha256": "573af0f4d15216b3eab0a7651b7857801621c6576933c539f2726f92534ad895", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.7138833759818226, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:43.525082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "9a1bc97c-17fb-48a7-9897-c4dd506c6a71", "result_sha256": "fb89d60a63505beee6cbe31249dadbfabbcee54b88c2c16d4a4dce1e6954f76b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.837022946914658, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:15.300580+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "05ceca72-37ab-41ab-8f94-00107eef3dad", "result_sha256": "31fd337673f4e5c8daecd3bb1e31d9a20a890afaef503c653b7a91664e9f516b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.883643456036225, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:28.918848+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5ad4f68-1111-4fcb-90bd-d5a8f061bfc1", "result_sha256": "353eca176aeb56ff9c463f258300f4ce2967076ffb43a47ae5e2e33d99460761", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.006069257040508, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:11.451719+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "28361790-a880-4e74-be0c-96602dc04018", "result_sha256": "9ef7fd6d250e62c660614c453625064d6445a0586cdc610ef5b454f690e6a75e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.9165299449814484, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:07.431956+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c3cca72-eba5-455c-9872-b9553664a0d5", "result_sha256": "b424bcae77af4d63952e009886315415aa4ae89ba7cdd336a9eada34e5267a8a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8395726840244606, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:20.698404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3acd3d06-73ef-4e09-b3bf-b8d415f2bb21", "result_sha256": "98229b4b0a56b72ddc7acd5df344f0c8851f0c91971cb5656273e100cfe108f2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8866891589714214, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:43.634521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "fcec1934-13e3-4aef-86e0-36bb05b6ff2f", "result_sha256": "25a28eafb93079f133a4f94eaa2a3f86b258c531912a0a984af3ba4e0fb2f977", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.9647500870050862, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:30.486343+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f3025d2c-48b3-4bdd-abb8-64ff34e9df70", "result_sha256": "8b940bff2cbdfca758391d091ce119877f005f0977296e4183ae3f05559d2f2d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8560610719723627, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:46.773420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b819de5-649b-4590-8488-d8c3b3b3e738", "result_sha256": "5780acf87dea8ace569df375b3622b8aeedfcbca268d21e7a32414ef6a42d480", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.9104911210015416, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:23.005847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e91eeb0f-fc1d-44cf-82ce-7ba0d7842f2b", "result_sha256": "24e5f681d45aaa46386aaad45135c80210a9c120408b24355d8dca38e65ba0f4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.1030064440565184, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:24.154306+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b510300e-1679-4bc9-91a8-0b92b43e1a81", "result_sha256": "c54fefc010916013fbe16e94cea3187840f804e7a215d22fb6e236ffdc6a46f7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.847441611927934, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:11:44.527403+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f6c2a3a7-5fe8-43bb-81df-6fd5c3598ad7", "result_sha256": "9c61ead1521395099313575d4a2c71d152a658be647234fe117cb6366b87a971", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003615, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.7910261900397018, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:23.883685+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e4beb3bf-cd90-40f3-b2f5-ad2862a69078", "result_sha256": "33e0f0ecf69f6b1ba7d89bb88053f6a349cba36a9832d9085e72a8cb7b7b34d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.8540387339890003, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:05.397754+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "905911b0-d0b9-4f1c-a6ea-a93ace0c4ef4", "result_sha256": "a98de08201ece338ef4ed7fe508f65c2d80b6e84d6a0a3527a7259cc5eff7143", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.083451158949174, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:23.380630+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "280fb016-3101-4146-9ca0-3627d0a2a6f7", "result_sha256": "f0a0035015be37c85965ef264a86ea78a6e39ffad4f1b0cc2a15b83d92cc9976", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.8688003710703924, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:09:39.383863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "99e2f555-393a-4641-8da8-037fa1c8b5b9", "result_sha256": "526bd516185e4a2b22b72e82180c126508ad1d24e4f635aea19ca4e4c82552d6", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.8809923499356955, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:47.639222+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a047567-9f7a-4088-b6d6-e439240f505e", "result_sha256": "f2079cee8aab89f791cea82069a4373f2422e197bf14f1683c6179a43aa66d67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003615, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.0073461580323055, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:10.976386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "91a8a52b-7583-4b7b-afd9-f01d436aeb98", "result_sha256": "59e879c649b2b3659ff0f4493e4f2ad024d4cca7c60b461598ce5116092043bd", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.7820789720863104, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:44.800764+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "5ac9ad66-dfe6-4045-86ca-e3d60d7c8346", "result_sha256": "7deedc7c8f26485802e83b33129de4642e6ec1c28678b360792b036d1fb73e6b", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.013740379945375, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:38.780159+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e57c3220-7ef2-432f-ace0-1c71ef476d31", "result_sha256": "b20adb93bb51b53765d376510d57f9f6e8f4821a99850fd23d7db936a8eb12f4", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.8638363090576604, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:00.547997+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3b669e49-e6c8-4277-94b6-11b7e98b2c36", "result_sha256": "46cca5d7a68faa00385be96a39c4532a3bdc0cd13d2340c231f5b6682a94a3d3", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.4310491259675473, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:13.601101+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c03119a-e009-4198-8fd8-ac7efd2f1560", "result_sha256": "5bd95abe3f93eb7c01be84b0d02c7087835c07eab16d3ee112f0aaec0963d17a", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.000374, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.7736874680267647, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:17.428440+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "938f4320-fe5e-4bc9-b1a0-d7591a335db9", "result_sha256": "c04839baaffea6dd2cdcb214ebd5d1f09ce0f6b2ecc7ec3f9a30d732437e9b49", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003615, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 0.8909002239815891, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:36.352317+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "97dcc648-c912-424d-a212-835a4c088296", "result_sha256": "2a4c54d87163142bbfea3f4c5a8836bf9c2fb5d2921fec8eb20997c9668b65ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8982682109344751, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:54.994762+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b13d2e58-aa67-45c2-81de-393657114be9", "result_sha256": "5ba5f7405735ef16c2993c44da5898c697956e576477795d0ef1668adf6bea0b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8400070160860196, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:01.256136+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8207bb8c-0756-4b86-ac84-83a5154a86d8", "result_sha256": "7f67d56d414ce36cc1b97a8ba94621d21c3b4772923bba31b81763db48c50cf9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.6507097979774699, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:23.212294+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf310470-aa97-409a-bb5f-d747e28d21e8", "result_sha256": "7ffc3445a065edf5a09c51589ff5249539bd00e470aeee12e3c1c89c5df75505", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8605371559970081, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:55.008549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c7a2397-af09-48ff-ae16-577f99a34f44", "result_sha256": "d490e53b98de74a26d7af09c1092f5685d03a5eb9e777033df5d4c04c207e8aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8590206179069355, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:47.583354+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e3331aca-54ef-480f-8989-8bace3d30819", "result_sha256": "b5012cdb89a8e765f48a1386e8f5ebf98a56ec0d6a94e9389ade4620c0979bb7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.6242005520034581, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:21.640525+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf46394d-bac2-48bb-94b2-edc749376ee0", "result_sha256": "8c2db3f7c45089794bbe11f0e2fb62544cab7d6660bc23888b33f9c5a96d5a49", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8160021550720558, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:27.598402+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "202072cf-b29c-4144-b93b-563d2f93fd1e", "result_sha256": "1c746c999e493c13933012d20c7100aad23540bd973183cfc1ce4e0d8253b311", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.0171194869326428, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:13:46.434506+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "15e5008f-376d-4bd5-99ae-fe3b88cafaeb", "result_sha256": "d5b67702f928b8437362112c4b2c0d275cc3e84195c3d3d9056022b5d3d9abc3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.3992469719378278, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:10.973076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "db82b70f-389b-4963-abd6-b7938b9d00b6", "result_sha256": "06804acdafcf232be82fffe12880112086a6ef9ee87b7ad6713c9338943219f5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.1878381250426173, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:13.799239+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "1825c861-be43-4a53-aeb4-835545f6188c", "result_sha256": "13c23e171d3c1cb4205c91be74e6c899fe9eba8e85c711c1ca7865f07690a36b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8049801250454038, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:33.961417+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e089df77-ec50-4aac-bc50-e104fa51c82d", "result_sha256": "9db22b1b9ecb65f4e8fbb906f201f9f709b3d44eb4e047784e450a5f06cb1793", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.2238846860127524, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:13.209375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0514384-0c07-4276-82da-3f3a5bbe59f0", "result_sha256": "37b9ed059109a2226f8a4e2e5503d0946fd90d1a7ccad1c8911ce15e63c3d3c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8103464629966766, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:46.539812+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "aceab2b6-2222-41a1-8890-74fdb6066266", "result_sha256": "eb1be2275f938f55a8aed25673c78ce3f327c7e777b9043193e62fb121fb8ca5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.106446776073426, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:34.665982+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "59e5b348-e566-4dc5-a579-e1106628d0ac", "result_sha256": "ad51835f39ad3666511b63597b5a53b3fdda914d972589957d9e1c80ab0c5291", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.7423271039733663, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:39.212543+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0376dce-1c30-4dd0-af3e-404f7bec97f0", "result_sha256": "b05ebb1996c4ef3a0d1d7c549b3887effe476cce5b558b0a6184fd855c4f17d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8289966660086066, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:33.671657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff93b410-6a6e-400f-8ffd-22306cb45aa0", "result_sha256": "62eb746b83ae184d5f81a34f78e029a242e31029882f1c693feb82dc6e0281d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8193403850309551, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:13.207507+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c917e3f-70b8-458d-9e31-ea1f25087f00", "result_sha256": "4d03fe9c8d2f02a0f621f96ecea6c46b38e17b5f1729ed64db2d49ea503548e5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8336727099958807, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:15.726994+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "817cb3a4-25fb-4e4d-a00f-97fd07001f3f", "result_sha256": "b4a24d0e37beecf2f6e04f59dc675262da68dff6ee8f382de9fec5513cfca30e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.328264475101605, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:54.764126+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3ed7ac82-ccf3-4ec0-a333-11e7f0a56dbc", "result_sha256": "de84e39868eba1723b594a027fca88a4842e44a17e9a81b6117cc0cf17911cde", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.2158444750821218, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:47.362306+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "62b314e5-d5ee-4fb3-bddd-d8a2c05cb9ad", "result_sha256": "a640761b67461e1946aa836b4ecee4478d2ab3bafd99108235e57232fee6f5f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036859999999999996, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.9777489609550685, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:29.908900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8dd64f69-35f2-4a41-b7c9-0b131da27d44", "result_sha256": "9f548c2e7aecf50b55600b1c447d4dcb81dae541a1130f047f167c2b6cb0f4f5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.8969985359581187, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:05.903288+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "643e6a45-ca88-440a-acc7-92cdaba0f21d", "result_sha256": "5f0d4eb78bbff1c9188fa98349bd8db0259e60ed02882ed29b15b3e305ab09e6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.9498083159560338, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:38.710287+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "40c4c324-a436-44fe-b589-e1d82f9fc009", "result_sha256": "de903d0181202dc7154faa43fa6bde286304c769bcc337e952fe3c008515aac7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003561, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 0.7538511659950018, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:06.786733+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "7879827d-461e-4ba9-8701-324f2eb993dc", "result_sha256": "7a463c4f44d0f424418c853a9752c3a53b23f469c00d7e095ca7341786954502", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7466606019297615, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:22.665712+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a5a37882-afd3-4fba-a687-a7baf847cb11", "result_sha256": "9afdc1ec6c9f04dc16370dbabff68f42b78bbfe26a502cb33a0aadbdd178367f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7818345410050824, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:30.077120+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f7507d9f-a412-40d8-920c-1a8e18fbdd67", "result_sha256": "347bd0f46c37a5286e77805fd8f8ffd290ab769b67c157601e275c76ac8e89aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7394243570743129, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:29.492136+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c588467-a513-4870-bfca-44fada68143e", "result_sha256": "fe517e2be691b0b9d91c39122dc0ed8156ef6e4a866a9bbbb03a36e33eda3053", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.0732390220509842, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:21.587069+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "afd5d749-0b86-46a9-8f69-1c2b567a7f1e", "result_sha256": "f9ec4f6d4fde7e67ac81ea8ad9d3aadeee42710d44e4b467c4a4eb3dc427052e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.8754926429828629, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:09:04.413082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a89a02f-aac9-4d3c-a0c6-41667f348e6b", "result_sha256": "3190f168fa0aee2f72e89a79bcbb2d3fcdfd3c13d4cfc4c9b735c734475029a8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.8169172019697726, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:04.191563+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "261f0187-a0e2-490c-9b3d-c4e268e8dd3e", "result_sha256": "7428805112a015a46811c04c50f9ffe87f5a8c8d7898545c71dc2833f60cb9ce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.1084294210886583, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:34.920046+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ae61d69c-8543-42c0-9e9c-caef1c7705ac", "result_sha256": "60208064db3b39f03fd82ec91dd6f1505dd647975dd7d4bb4797d8d13a140c7c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7340271410066634, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:24.584450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8e4d304-293b-4475-b8b6-0d40f7ef218c", "result_sha256": "f1d4fe2536b3cfd1150a3e33828bf3fcb23462b9576105ef5febc6c92b59c6ea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.05435878993012, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:03.571675+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "38c5a150-743c-43f3-a3f7-f7be17b7077f", "result_sha256": "32b0dc4a1532dc0c78bdae61d2032c72c9f279b7248ab82dca1320af59bf76b7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9167336199898273, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:29.905009+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0f73be2-ac8a-4d09-8c8c-6bf1c0870ad1", "result_sha256": "8e5dd312b81d9dc9f7a1dbc048bc537591eab2769b4253637e1ab32b100a61f3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9204727009637281, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:18.410100+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "fd787054-2143-4d32-b7bb-f18ea8217ac6", "result_sha256": "21fa35d7fed7ba46e16f89e653338bbcb20b8389df7dc0580c68ed4c28bbdd5f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9143505740212277, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:21:45.045473+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6445c783-fe2f-4bed-9cf4-6eb7d086f233", "result_sha256": "d4ee5edcd9aa39e8184d210e1365f79d4293265f83ea42f4fb8a7060638330d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.6710527880350128, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:18.553001+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ec99d21-4de6-49a2-b6aa-e0e2b2a2702f", "result_sha256": "bbe952e2376e1e896a0575baf5cddc7504d3f2c5f6b4b72de48ce79de2b0623b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7352792739402503, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:19.635472+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "dea76067-5eb1-4d8f-b6bc-8d5d1e5e3fe6", "result_sha256": "35d55a9902f66148d28f9c1e6039c913a61bffe89aee1ba2e159ee518d4a2471", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.2062135859159753, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:36.990636+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8660b526-3747-42df-a08c-de20ae845de1", "result_sha256": "8325938febc072d751fce5f68fd677bbc9673a060cf2427f17836d6de7438661", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7693761099362746, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:07.238917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "98cd321a-7226-45e7-8a2f-92a60b61dd55", "result_sha256": "698d3d307e2b6d342fbebe1269bc9abd3c7b51c2fb51e1dfea5a5274b51523fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.8219655540306121, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:23.292838+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "36967e90-50e0-4ee7-bf42-6b10acb624ab", "result_sha256": "ee43850a52b02b8666b55ed3445ffa7e0d98ec7e245344f60401c13796ed5108", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9217612019274384, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:31.874703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "c87e9790-3a25-454b-86ad-a84a57215480", "result_sha256": "bd21f017100f1ae5c1e6e6b98f7a84b6ec174f5dc75f38da299bda74ca1f93ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.8059873620513827, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:25.074000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "d33146c6-3c1f-40e0-8391-f64df95db394", "result_sha256": "694bf17d78b5121797c1b16fdfdfdc0c84c1573f5c2ab70156503df2805625c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.7198542709229514, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:12.212488+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3b8ed48c-48fa-46d1-aa71-141e46cbacda", "result_sha256": "d3055b9714e63d0afe1fcfec59439b64852bc9bc7e4c124874ecf1a0b844c391", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9908346260199323, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:47.685370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "61145a7a-d3ec-457e-8ddf-d3806026cacf", "result_sha256": "3c03a3dbd5c41cbe2c68c80681c1accf61a03a6ddb2b02d7e98932111a4308c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.0026776890736073, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:59.312257+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "10a69016-d841-465e-a147-29f8b9ba2ceb", "result_sha256": "59a9252ac6212c83e7c6580525c495b1995b7ee2472d3b5d505c5225ea575ef3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.8213758430210873, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:29.589411+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "c933414b-7f4e-4b23-81a8-da6fff8e3e66", "result_sha256": "17486dab78f573d547a46109aa1ef2ef8984a6b98baa2aabd851f714f8f826e9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 0.9019740830408409, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:49.794784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b220b220-97a7-45d1-b81c-05b1b4b80aac", "result_sha256": "ad4eff25611e10d38a3b393352de94e1ba6a79d67f0ba2a90bac38da80338d2c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.8366459099343047, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:58.812478+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e44c8f68-36e8-4481-84c4-d6276ff60c52", "result_sha256": "f1470e64e3c53c171f55f695187991dac854d44f5d8728e0de3e436536d91007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.9277273330371827, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:19.778832+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9f0dae2-f77e-43a1-a9af-fdee2648dce8", "result_sha256": "1a8fd855084feac856698349955210affbd0666acc9f69f0df27d97138bbe20d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036359999999999995, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 4.047107063001022, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:15.650337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "be9ccf73-7121-4514-8a68-9c036332e76c", "result_sha256": "3a5445c46fd4671edbab584c05bee1897c8411808e51becc9c35d8ebbf5363ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.7955767150269821, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:08:47.611083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "eb6c7fc0-a01a-4f06-9003-d93dc40b2ff9", "result_sha256": "bdad2807633581daf3cf033974f0e895cde0299a92478c77c26fc4d82e34127f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 1.3650587480515242, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:28.257961+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "321241db-96d1-4378-b99d-c7616561be2f", "result_sha256": "09656551f47a1988bbaebd9d072464fe0782257398d28a69f16340ccecad8060", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.8968357000267133, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:33.134226+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8a1e5f75-0d0f-48e8-80d4-3b144fcb62a3", "result_sha256": "e90cce832130b9d84dd68a3cab5097c5f1145121283d0f96114b24677d23908a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036359999999999995, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.8521636930527166, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:14.489677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "83593da5-1a8d-4ed6-88ab-aa6304128c9c", "result_sha256": "d44f0e170ce726fa6769660f4c6e842e674e86aecdcba4ffc85eb814d13243b2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.9148807669989765, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:44.742342+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "c28c5df4-7840-44c9-8189-b9e57b04a178", "result_sha256": "77edb3972e33a80e95d4d4e6310c4f6643495ae0017cff3476e84c7eb843d19d", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 1.5356252730125561, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:17:39.945206+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0fff9dd-6061-4822-9398-cc438405c1d5", "result_sha256": "bc052ee60e5ba53974065accbb4e089f490af3aeeff09d04404a5e125dac85c9", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036359999999999995, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.8213586539495736, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:48.334597+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "101ebff8-ed2c-4b2c-b068-de54fd7cc5c8", "result_sha256": "972e70a6e7757e038137649b037cf05d9f326b3b6b65017f78b9944158a8f854", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.7040724960388616, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:33.798211+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "0b4146f8-ed18-49fc-955a-8813722e3544", "result_sha256": "8acf848fe11643366f5551f82b5536f65e346dfc612e622f0a9542ca89560916", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003761, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 0.7934254489373416, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:01.825206+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "90891982-1a60-4e02-8eb9-bbd832bb5648", "result_sha256": "82680fb7c27f1f090fa33bfe9a523d134005d820b35aaebf128f537135e951c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.0415956679498777, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:25:51.770334+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "15b5ad17-3ef5-4883-b8d8-00dda9bc878c", "result_sha256": "061ade64c1d27ba2a204bbe180584cb0665fa983f52ca9a6fc44f592f6e74908", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.8124513559741899, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:48.295794+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "cab87e6a-4671-4151-b0b7-34e2902af5c2", "result_sha256": "f9b009d0195585a529c35a6b69c71da815992ef8cabf598cc0ac6385632e0935", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.7741560710128397, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:12:49.403500+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5bcf644-124b-4597-b5d6-06deed02b389", "result_sha256": "a79414e6e688fa884a82246e488841ceab335e0cda1127c47eb2816e7126ebbc", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.8020970849320292, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:15.520901+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "c24220fe-adc7-4acc-b04d-1e924eda4e96", "result_sha256": "45d96a7b7f529d39f62d369100dd850a6582060b7d15ea6aebc7b4ec047007df", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.901230383082293, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:51.516835+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "fcd7c131-e656-4485-b937-5d2ae396e662", "result_sha256": "648408001f240e69828e86bc0552f116bf99c4ead4502c47f882227f615168a6", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.1145819310331717, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:39.238368+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a90bad8a-830f-4d60-b08c-63a44e5b6940", "result_sha256": "c9ca9f1e0a448cc7db23b62c516e19b08be58d66d6e74a9c9211ec76d3d25d27", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.9758359850384295, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:18:15.522911+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3bb6db6e-0dbd-4e37-a600-51268f5b4307", "result_sha256": "04b4db44f6d88a808f8020c5c405197b7049058548592888342609a632f677bb", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.8353924880502746, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:40.553555+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c41fd24-2bf0-4059-8344-24a6185dcb70", "result_sha256": "d1baa55bd73c05ca8fbb00cc6d8902bb6de2f1f78200cdb3bf4915fd1a3af364", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.062646355945617, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:22.679517+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1b11618-aabc-4f85-bd51-e412cca3470e", "result_sha256": "c4c1705935a1076f75d9a9bb46800ed88c1300b1148b66d2912f4f4ad9fc72b1", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.8954552749637514, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:54.250617+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6dc9aa5e-7c91-49b3-bdf9-80c7473a4f91", "result_sha256": "cf0fa1180a826e89e4c0ccf0aecaa2c5cea6817efc19f34dcce1dba3ee09ebb4", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.2405230150325224, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:04.104297+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "31e59e81-e364-48ea-8ff7-8515bce11e42", "result_sha256": "728e9e0f3809132d9a49a29edecf073f5786b9bd17359b8063b3f8716e155147", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0003764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.9951251980382949, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:30.050478+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "5d8ad0df-831c-47db-aea1-8a439ef2bb51", "result_sha256": "6d0ba46b4f47af23a006cc4bc139bc595c629bf165c3db9db21348599512b96e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.9688667790032923, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:06.118891+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c36f5fc-9719-472e-9b4d-2b56e8cc20df", "result_sha256": "e2aa6011d267599bde8b5d00fde2fdea92d3e2713aeec347ab5a306e46fd4c3a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.989675673074089, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:08.318673+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8833de68-2660-4aea-96c0-eec81dd4f16a", "result_sha256": "ff56a3aa888a9ab0b6024a1d98b18413e0fe066d444b3020ade44bdfed59a533", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.9608224689727649, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:38.741652+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "18c529ce-65ad-4d6d-b832-8665b1f9cd49", "result_sha256": "bd1c8cbb18e9c9ba38c7c99bb8340fb2661f189ebfc751bf3572be2fce33418a", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00036389999999999995, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 0.7385731369722635, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:12:59.191766+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "9a81cc99-4a99-4d78-aaa4-02b6da38e907", "result_sha256": "781773865c0dd14c837015223857ecc9fc7e6309774ca9a156c0aa00d985e2c7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3101414470002055, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:53.115868+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "278f4bb5-b93a-4975-adda-2526ef3308c9", "result_sha256": "820a81eba162930cd062707153890b21752ee5f768362b0817199a6c62d687e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003495, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.8556215089047328, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:30.826955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d2d765b-7bbc-443e-b1bd-20a22119dad0", "result_sha256": "0befbc49cf95cf21a031d4b3708387d706bbbdfe316ba6c0ef16fecf18650c20", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.8720185570418835, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:17.452422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "61cc5e3e-e453-424c-b791-a80538207fec", "result_sha256": "ace79ca7286b208e300e37260f79d4170e5647d6c8dcb114f8efec140911b76f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0476396949961782, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:27.753725+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ec50a3e1-db53-45c2-81ee-3975d0ad07dc", "result_sha256": "e40e9b0d9d35821843b0da78d5e584fa53376e43583a4adfbb63efc431081670", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003495, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0054808240383863, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:26.958853+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "c418cb79-1ac5-430e-a212-f1d3b19a6e9d", "result_sha256": "d74d99e4d586ca29d0c202cc0585dee1ac3dc95d484fb4481bcb068705c7087e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.8803062109509483, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:34.131887+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6ee27884-a2c1-46d5-bf59-8466bf82695c", "result_sha256": "f0073a03471c67cd09825129581bc804602c2c8d728d1af98fda1b90bd197a1d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003495, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.7687453529797494, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:08.599966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b9033b21-46a5-40a0-981e-9dd3f5eaa754", "result_sha256": "b773c011874a4ca502fae8d81616925d9d4c11c46973c56299cb63b2572636c5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.9022914699744433, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:48.308955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "4e65020c-aaac-400f-b360-df9350b3581a", "result_sha256": "08164e051ab4067c1e07cfe898d27be8c83234ad7d6fba06e047867ea2ac5667", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.1774505189387128, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:01.749691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "db9768d4-83c8-469b-8d5a-1d08e43da4b6", "result_sha256": "f54562dee1014ce3b2dfb0c78772a9e8121b6bd830437d693bd6c2419df9124d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0089574550511315, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:10.133886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e157680-b280-465a-ab71-506cc0829d7c", "result_sha256": "a7b64f851b8688fbd4eefc8ee2ad7fb409d70382317e74687114dde2f1a262f7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 0.998752381070517, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:48.085649+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "fdbcfde2-32f4-4029-a2b0-1eb0d93d6a2d", "result_sha256": "9edbbcdbb1f70d95b85419a7804e78de2944031b0f9dc1976b3e4b45cd67018b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000362, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.0622982999775559, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:58.584878+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8a38b735-e714-4f9e-9584-ba18d64f2f6d", "result_sha256": "5bec969123d06857f050a4d930cea6147c0546fec5e27e1d8d4e2bdd31ffd2ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.8227858360623941, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:21:49.335999+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "af0905ea-f343-49b7-8b2f-b3b979d88591", "result_sha256": "1dfb30a7fcb14908626701798eecfe574486a273c32251480aac999372be20a2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.861596364993602, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:14.554656+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "aed8e89a-3a13-4783-8e86-f6676f671f06", "result_sha256": "00f8278710bb419339a546e98cf40a0d38b0ba8566545a87c5a4b9868068750a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.4630061640636995, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:05.606891+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ef635988-b4f8-4206-a150-0035ed736e56", "result_sha256": "ee49d4f412c732919f75c92585316eaa3f7b887a173ebc36093468c91b78b9e2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.392915635020472, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:08.954954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "93c7d26d-369d-44d4-ad9e-3bec6376aa87", "result_sha256": "e8b38f0de886a37898d08751cd8ce8b3109289cc4d75464177958599dd0e67cc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.4193870819872245, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:22.115103+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "78597714-3b37-49c6-b9f7-e71e778fd988", "result_sha256": "491703d7c15e8d902c3252c91c3885a1d0e15f2949498ae00047c0affa65a7aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.9607002430129796, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:13.085489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "1308da16-04fb-42b5-8532-be84178f6108", "result_sha256": "23a4d3f3252e21817d0d9f8e294ca5c4f96ab91855361d46de1a9a1f275fe3ea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7556012640707195, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:42.057623+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "8107fb49-eca9-4a6b-acfc-91fa67fd9116", "result_sha256": "5084060692ad016992eb11032ceab5727a1aa1c3930686d3b2d9ccf72f3469dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7659274289617315, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:51.261268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "02882fa8-430f-4560-a458-5f00327d815f", "result_sha256": "b5fbf7e583414ee79439ba45726383dc76bcd1a99d1acbb3e79f9cb85476d547", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.053692044923082, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:25.876041+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3bbb22e3-3966-4ba1-8705-cfbaf2e0a1eb", "result_sha256": "753618fe813fe5af4dc7cb6386c8a402ca701440f36fb57234baeff1e4425af9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7489059220533818, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:27.592828+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "19953d9c-485e-4295-9492-36a26d52f294", "result_sha256": "83b4167c86adcd71a2f2bb2c79ad08e84fd776f6891d60a51cc9c327e44e4b0b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7972928069066256, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:51.779457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "4857899c-2eb8-4322-9015-830b41a52280", "result_sha256": "818cfff6ab8a4bb83ea8a41f83517ab57fc909cf50137780e1be729780c55de3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.9579442810500041, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:37.784195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6abb5e43-75cb-4d07-b136-63ceaf1e1fc8", "result_sha256": "b94ae5194f3c881425e69ff2c800c4238150233fac5f7dedd5640327aeb6844d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.788845579023473, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:10.489330+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0c6e9be-8619-4d93-8be4-e1d33f077367", "result_sha256": "00babc9ce67a9fed82c580103b1b7f51b8fa7686143c56ee384f04aa7c8b6957", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.0899517909856513, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:37.059689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "95ec2330-7b2c-4411-8f54-f5ce2172cd68", "result_sha256": "a322c4eb2a7704df0b066820f8c175bf25f84d997245175d755b95ab035e4f5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7014452690491453, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:07.868963+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "ad373621-bde2-45da-850b-50b1b3cf8bc7", "result_sha256": "8810b7a8550173a2be3419e92f224de500e231a2563291f5aab75d237bb308ea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.1162010510452092, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:19.068426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f94eca5d-d1a3-4d2b-8bf8-cf0950fb1642", "result_sha256": "90ad854b10456199f54e5723617d0df3c216685cb430037fa0b18e48553cdff3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.781163937994279, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:27:18.820648+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "155201d4-107e-444a-903f-0a1a05fc73d8", "result_sha256": "cb013efc8663c6caba56e6d1e63ccfb6354847455c95c1786b350e22ea9d6046", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.8978101019747555, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:43.789008+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "247fc307-10d0-4b41-ade3-eb2fbdd953fd", "result_sha256": "4650b8c18693c328bd28c531ec8da416601363cc903b2cafea6141c686b4a5c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003552, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.8632639140123501, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:57.377141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6375fcbf-0356-46d8-bfef-2ad51bba93b4", "result_sha256": "a015a8b783db6371408425b55eb5b401eaff592f815c7d5504d5f27ea20ec40c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003552, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.2309671400580555, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:15.897506+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "79c5c9bd-e819-4763-901e-d8a86c5cf6f8", "result_sha256": "c67ccca00cb422d28b94a1e921eb91b4bd5d8ba3b74f44099e521928016dff92", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.7938476919662207, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:21.489061+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "19c8eef4-d28d-43c5-aab4-348b77382444", "result_sha256": "f6acf5c2ef53384b5672d1a113d0a12f4f0b3e4786229135e0f839d77143e685", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003552, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.0564438719302416, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:37.332765+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b6dacf1-1571-453c-a17b-602285951b24", "result_sha256": "a2b24840a6b48cc5a4596d99cb291e7dcb177c991ee932e30dfa0536cc89d80d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 0.6507346229627728, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:26.333678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c1cbda9-26d3-4d60-807b-fa6bf7d07832", "result_sha256": "86832161d35d3022100898ca8a7e52332d02600918f5ca61789e766143bea726", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003677, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.5045402290998027, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:24.919386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b69334d3-c0b7-4467-b408-c5fb1caafb99", "result_sha256": "aedef36064e3c0a8bcfc433e658f613c93134e7ba0290ecc2344176347184e11", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.0666452830191702, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:36.782791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3963a5aa-c14c-421b-96ec-e62849437c27", "result_sha256": "c9f3513ff45c5a298f17eb47b8b65f655801ddd38bddf6e5873406ae0b3f77be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8563484740443528, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:42.414809+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2bd199c-d190-41dc-ad50-ec6000b4888f", "result_sha256": "84c1d1fb20078ab170c5eea5831e3447b49a32adb0e796393e5fb39ae252381b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8544028759934008, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:01.585599+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "0bd30c40-b3bb-41db-b501-99bcebfe0ef1", "result_sha256": "e07294384bc640ca9705dc157c03aca7ddb3ee9d7ee908d8f163810b2a9fe304", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8450173130258918, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:04.484535+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "9265ef9f-7168-43ee-9df7-eadf8f8a543a", "result_sha256": "3c4cf74c8126c0a3fcf2e8fe47566542400d3dfc5d81f36c23dba4a26f7d456f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.9949227250181139, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:15.024122+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "bb247ae8-b660-413a-9512-541e3d7a3a5d", "result_sha256": "df3f618c143aeae46072f3f91194ec0db70cc6adf2b2b662de0354d2c237fe25", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.0269477660767734, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:21.814142+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e3cf3862-94f4-46a1-a280-d1caac9233ad", "result_sha256": "e5491a8608ac7ddbeabd22fbc0602c34adbb2fc7deda3eac80adc77e75f1da36", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.040268320008181, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:03.148726+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3b8ddbe2-9a8b-4cda-aa24-87bf949768be", "result_sha256": "e5761a42a4ed71a09534cdd0090b90aa5a7502cb9438ef5682fbdfc0758ece60", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8774744580732659, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:18.618635+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "918e6e27-d1d8-4247-84af-4c6a85e7ca65", "result_sha256": "42b7ccbb4193bafd5d7ddaaa613c47811dd069de41fe56ff16ae4f20d2ba5305", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8905846739653498, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:59.661029+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "19b98fbb-3421-46b8-b218-67e4bb25c211", "result_sha256": "a3a5a9e9774ef7436aae4c1abf4a13bb271b74d4a75458bc760051552361d8f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 0.8582249619066715, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:35.441732+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "23a05100-3b29-4d79-8100-7dc97e6c3705", "result_sha256": "92701fe1b88b04ceab1baa9d1cac499ae3386717ea8a08d640bea90b3be27e42", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.1625477219931781, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:44.211724+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "791fb461-6936-4262-9e24-1072aacdd511", "result_sha256": "feb78c3d69848183c1ada0f1505c531083c998bb5f25683644a5c62ca17e9f17", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00036559999999999994, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.3593448960455135, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:16.073916+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ce6b080-c625-4e80-a2ac-a0b9e709e7a3", "result_sha256": "9fb271c6b94115f7d694f7e7d8b47a26da9d58c8078d69d4cd77682e062e44b2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.8184933529701084, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:08:58.283671+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "50aa70cc-0a42-4ad8-8392-c4ffb1b22ab5", "result_sha256": "7b9def3329ae4ac94123eba42f285bbbd449758d67ec849337174d96588a7f7a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035580000000000003, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.9037415160564706, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:33.595354+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a68e611-3321-4983-8a23-83b91219f079", "result_sha256": "b0fe7f8b6eb1a4bf2244d52e788824d8ec4cb7d1dc2f25512dccc21420054672", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035580000000000003, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.8060680039925501, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:59.386679+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "041b85b5-7fde-45ac-bcdd-5a21940e8e12", "result_sha256": "1fc38c76b8d870aae97d82ee0f6f5985c612c0b4410f86241efbaa01e5f9dce6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.4749060830799863, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:36.069612+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "411a6406-3850-4bde-8518-d51daf27c33b", "result_sha256": "1a1de24a712c4fb19c83066aca680844e1bc58a92f2a3ede935140f97068c734", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.86955233104527, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:44.932816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "30a6a4bc-4144-4721-9baf-aada470f1eab", "result_sha256": "25627d9bfb18a2817a2ce2ba2ed9ce37065bcada6d50da78ef73cc414b3dc544", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.9957898210268468, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:41.342955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "041ded65-b41c-4eb2-abf8-7a9366184426", "result_sha256": "0ef2e72a6809c7c69e4a9c21bcf797b8c141097a01bb81c701402a07916b2850", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.7668445679591969, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:53.003845+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "bb4889bd-db9e-4928-b722-46e1bbf30ccb", "result_sha256": "cf7a016067aca712304e4c4b0479574829c58d8326be3b614228053c5d8d9280", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.0621369431028143, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:35.916677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "629f21c2-025f-4206-8f1b-d97e97d7a1a3", "result_sha256": "83d684339a9d5518348725dcea39540897ee562e528f816c3086acce9d4a4267", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.7094053089385852, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:08.422924+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "04a2f249-05e2-498b-a048-034ce8cfc7b4", "result_sha256": "4d070b787b45b984a0c7862dc897d0d3e77329dddbe3a3fec8de6e0cb8e25843", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.0627019400708377, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:11:57.084078+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0fdf564-ef82-4532-93d5-5927acab911f", "result_sha256": "3b967d516af4d234d46172f67363ffdb004dc017237e056e85b25f1daa4c77da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00035580000000000003, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.0131515270331874, "model_name": "gemini-3.5-flash-lite", "output_tokens": 6, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:43.223837+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "65c4150f-3b69-4271-92be-c34e96c14223", "result_sha256": "ba8b0714bc3d2bfde67f971af7f309f309771b5a382e924c9a762dcce7c9dd15", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003683, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 0.5341877520550042, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:52.316850+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "580b5c11-a3c5-4195-8d53-821867d9484f", "result_sha256": "19bf10a2f1ec4a4c0c15e3ada66adf5078aa26d872843763032babbb34528a38", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.9173841370502487, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:06.464294+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f2efe97a-3deb-443d-b8a8-1a2f9ca9b9c0", "result_sha256": "11f138884bacdbe1c04b4b61ce147dc7a59e501e4e9e58cb02931d3426d973c3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.7055422390112653, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:32.647353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "f531ea31-9783-4387-963e-222ce1825823", "result_sha256": "f013cddabc35cb20a5c70e445a1f13df4063b03ac320e6e6f2b2576b1728e68c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.9461855919798836, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:50.735389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "e6a6ff4b-adf9-4ae7-b62d-be83a9c9cd3f", "result_sha256": "579ff090af390a47720b7470f6439b307eef715ef08ac8169450b5075fbca64a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.8204549259971827, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:11.112560+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "29e9d558-75f9-4897-9340-514be3b014fb", "result_sha256": "4d05cf84ec23d09fc9d3d784f07df6719eabd8f653f5725e07e21fac20434d5c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.794377721962519, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:18:04.105499+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "11fdb403-6858-473c-acb1-e3712c6f5e88", "result_sha256": "bc8790a3fc4507b530c219439bab072a0afc9cbaa3229bbebccc6c95fb504e80", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.8775847860379145, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:51.434669+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "b9c81e8a-009d-41fa-ba99-2f9269aa1d7f", "result_sha256": "dc32bd8c3cc05471a0be5603726d39492bcd116250f20009a976250584321578", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.0543175500351936, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:19.095634+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "3d8c2da7-dee0-492f-ab54-b605de7a6fa6", "result_sha256": "4730e17bb14cc6b504d671f406a2433e45d8f254b7565e6824571537aae3fe9f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.5009731630561873, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:07.154011+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "61862baf-4eb7-4053-8f73-6c64c90697ce", "result_sha256": "affc1d7cba74e514d810f0e42ec642c26758aa7fb2304d164db2cc60d705cdbe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.0426319680409506, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:49.812789+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "070d7d8b-755c-4a4a-9ec8-1f7c0f0f34d9", "result_sha256": "32fd945a1956efffe260adad28a14cf7992888e78adea95566099aa6c6c14c9d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.9958399450406432, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:31.437970+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "5a61d784-e304-4dd6-88a5-3bc90354d802", "result_sha256": "a71c5691b6a79556a16731a6670628f55b7bc30b39c5d3bd0ab1e897bfd4581c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 0.8460418169852346, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:18:55.693980+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.5-flash-lite", "status": "final", "included_in_comparison": true, "source_attempt_id": "a39f0289-09f7-490c-b180-3335a5fe4b10", "result_sha256": "b467d39cf15659be6e7e2e3e7889d658a8689cd52b703c3722b1b14b0c42fe5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0003617, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.0463262939592823, "model_name": "gemini-3.5-flash-lite", "output_tokens": 11, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:43.891776+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4bdb6cda-a302-4cff-a621-695c10222ac4", "result_sha256": "7ede2671abd36ec26db0b3b271ae83b3f82d3d90210215934ae10ea30230380e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00138675, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.6641497239470482, "model_name": "gemini-3.8-flash", "output_tokens": 141, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:49.591194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c45d54a-789f-4c0c-8986-a2a73146f04a", "result_sha256": "dbf1f4ff8be3c65507d4180dda14e83294cedb39281c733215c937395644d991", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00143925, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.9860886350506917, "model_name": "gemini-3.8-flash", "output_tokens": 155, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:16.654749+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d7f4f42d-0578-489e-acf7-f1f737ed3e59", "result_sha256": "43b05765388994c5e1f31ed5391c499402483d7800ef31d11161ff627da19bbb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00184425, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.7287241129670292, "model_name": "gemini-3.8-flash", "output_tokens": 263, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:09.699792+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d3eca3b-97c4-4ead-b5ca-5db1aecd8eee", "result_sha256": "759bcac57ae6e8e57846f40ec8442590608b0f9e8e4848ca279bdd949f2de3e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00150675, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.7214524009032175, "model_name": "gemini-3.8-flash", "output_tokens": 173, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:13.544943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "5142b8b2-0923-44f7-ab26-df1ff7551016", "result_sha256": "460c03bdb4ce6862147f5c32f2109450374ef57380b7e4d052e7dd9b51ff536f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00179175, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.978606307064183, "model_name": "gemini-3.8-flash", "output_tokens": 249, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:40.443786+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2a42914b-ba2d-437d-a2f1-f5324003d796", "result_sha256": "c1b024bee2e12233e243973e04907d1c14f3e61d0e179568b0508ceab759309a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0017055, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.4621168039739132, "model_name": "gemini-3.8-flash", "output_tokens": 226, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:54.827185+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8c7b060-f17c-40fc-b4a0-9deada5304cd", "result_sha256": "2752b2722760a040912ea5a05769cc3ef6479d3b315b496f8d2b1227c5f17632", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00182925, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 2.015661451034248, "model_name": "gemini-3.8-flash", "output_tokens": 259, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:40.424395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "36f33a0a-3c15-4d03-9fdd-f67a4c653c7b", "result_sha256": "0aa5b667627a6d70895783408011673f381fe575780a0237bb7ad5a30bf9850d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001878, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.7777441299986094, "model_name": "gemini-3.8-flash", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:59.154717+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "683c309c-ebbe-484e-b096-cdb134abd8f4", "result_sha256": "0a553e87826a3673f0f5d86f76e54218fffc7b07ae6e62fe519ad2b2992ccb89", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0014505, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 2.0745261419797316, "model_name": "gemini-3.8-flash", "output_tokens": 158, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:31.205165+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "90ebd87d-84b4-4af9-9792-e49152a47908", "result_sha256": "349509542ac005cf5e7b2dabf9bf49206b794ea24dc8388568a9ea5018b0faac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00159675, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.8314471440389752, "model_name": "gemini-3.8-flash", "output_tokens": 197, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:53.487371+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "3b43fc1f-ddba-4419-9e71-72b6d94107d9", "result_sha256": "1e3030cd3ca4a5346a36d264d266df32f50f3fe8bc3f3e36d220cc31f26c9a32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00158175, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.8479789099656045, "model_name": "gemini-3.8-flash", "output_tokens": 193, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:31.132577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "dde1f00c-ed51-45e8-80d5-34abc7a161a6", "result_sha256": "6ea4eeea7784b5fbd6f3070142c6c7a9a6c00141b08215bff504c31c26fab968", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00145425, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 1144, "key_errors": null, "latency_sec": 1.2699464949546382, "model_name": "gemini-3.8-flash", "output_tokens": 159, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:51.836535+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "8595e099-63a2-4979-9ee1-2ae4f6cd5975", "result_sha256": "e1782bc98a752d660fba72865ad816e0591f8fd8832abb091704bccb0206267f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00139125, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.3815137699712068, "model_name": "gemini-3.8-flash", "output_tokens": 129, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:56.196404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "89b3ade5-8d82-4cb1-8b79-7b0e22c1a24a", "result_sha256": "43a478072fc944b4f88dea6226369023406dc0bc0c03ecd92c7891c729796078", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00182625, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.5604766469914466, "model_name": "gemini-3.8-flash", "output_tokens": 245, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:01.498900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0836ff7d-8640-4405-8486-f66922cc7601", "result_sha256": "bbdef635076626d3517b215e7dc5d38d11a10ccdd1307f2296fee071a0a529b5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0016575, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.9831654580775648, "model_name": "gemini-3.8-flash", "output_tokens": 200, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:01.833514+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b03d99a3-5af2-48a4-9114-81f4c5d0bf1c", "result_sha256": "ee9b4b45d7b2f7c4fb0d390197e301717edd2e3372bc6be07fba2369d063ceeb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00176625, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.7102847839705646, "model_name": "gemini-3.8-flash", "output_tokens": 229, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:19:06.464088+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "bd93197c-ae43-4990-850c-bd862b518320", "result_sha256": "bb5bb766c97ce2c3082b79eef0dc9e63503dcf62a7335dbcc1d0fb586383f660", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0024075, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.7858624089276418, "model_name": "gemini-3.8-flash", "output_tokens": 400, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:25:12.346822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "3152d666-cf81-4d53-93db-c5ef1c8f71f2", "result_sha256": "a89d188d13dcfed3c1b2255da85954194777245d49a089c79cdd9463848b5ea8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00144, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.6664817900164053, "model_name": "gemini-3.8-flash", "output_tokens": 142, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:22:12.331448+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "03ca7ae0-7b97-423c-a8f1-971f82da7cc7", "result_sha256": "ba0b2e2b7fa6a26971e9fbfd0908fbb8c35337f2589684923237c2e962072ed2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0022425, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.6923715319717303, "model_name": "gemini-3.8-flash", "output_tokens": 356, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:18:44.215586+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "be53e034-6626-48ec-b54a-10bcc53cf150", "result_sha256": "d9531b0b4cff5feb74a705ab8065ce7a1314d63e4e78901f3f77e93478585ffd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00178125, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.6200488790636882, "model_name": "gemini-3.8-flash", "output_tokens": 233, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:43.199123+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f03d62d0-993d-48e7-8066-523565701ea0", "result_sha256": "ac8b5ef41aa70b2a133bd688c2b5abaf2da88ff958c7553505577f44e1d4c552", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00169875, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 2.058656213921495, "model_name": "gemini-3.8-flash", "output_tokens": 211, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:27.861819+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2a71f1f5-118e-4c89-b244-97abef78a480", "result_sha256": "0b520a8b732e06cbb990bfbcbd779bf3d4e27e40c564540e00c550288184f6fd", "result": { "correct": true, "cost_estimated": true, "cost_usd": 0.020819999999999998, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 2.4746212100144476, "model_name": "gemini-3.8-flash", "output_tokens": 214, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:23.179320+00:00", "request_attempts": 2, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 1, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "1adffe89-5a02-4f4e-9a06-6e8dd54dbfce", "result_sha256": "fc4b227f70c49d8ee987ae9b8774cf18f172befbad2e0290b5ba1ecdc8039ceb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001335, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.2954809730872512, "model_name": "gemini-3.8-flash", "output_tokens": 114, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:22:07.626770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "eb1aba02-7d19-409a-ac0c-542c3d7ff2b5", "result_sha256": "4ea84a7e9b8436e57525d0de2b2af5067bdec061682a9eebcfd24328e8c548c9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001725, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.653746715048328, "model_name": "gemini-3.8-flash", "output_tokens": 218, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:33.668612+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a94344aa-385c-4204-b505-13a6a5c1fb75", "result_sha256": "db4a6b839e0071fbcb40d44fe5951baf7151a75792e526a8fc670af22d940cbb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00175875, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.7359063269104809, "model_name": "gemini-3.8-flash", "output_tokens": 227, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:51.160435+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "7ecdf771-60c3-439a-a4b2-1f1890b1e0eb", "result_sha256": "80d51edca3bb88c34a3a3837520d976f3b440cae85256668bea0953d984fdebe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002115, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 2.103004071977921, "model_name": "gemini-3.8-flash", "output_tokens": 322, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:03.361477+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "bcdc78f0-54de-4adb-96c5-fb1d85fbe653", "result_sha256": "f6cd3fc6064828f8fd94e5e932bf43a8c5287121999a32d7530301915165f521", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0017775, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.649504448985681, "model_name": "gemini-3.8-flash", "output_tokens": 232, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:14:58.307171+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e9a0260d-7ce9-4505-9a28-ebc99221aa8f", "result_sha256": "d5ae7f6bf11e1a4a70c0ff713ee751f7550c91a52d339dbe383e99119b36e28b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00204375, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 1210, "key_errors": null, "latency_sec": 1.7940037329681218, "model_name": "gemini-3.8-flash", "output_tokens": 303, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:18.806151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "24971451-f8c3-4f0e-bd0d-283a176b68dd", "result_sha256": "f79108e448c3457a34cf2bfb923a02c69ea2a0c0045180b103d0b22d6ceafee1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001155, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.6964867749484256, "model_name": "gemini-3.8-flash", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:40.511311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b5594dd-3e75-42e2-81e0-5e1051d88c8d", "result_sha256": "2a0e9ba6642ae8c8ff4e7f8adea96f641d7d1d251970b46bf63186fb7a0cbb7a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001155, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.2642717739799991, "model_name": "gemini-3.8-flash", "output_tokens": 85, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:11.659471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b7982d06-155d-471f-98d6-e30351140ba1", "result_sha256": "8ffe3ec8520d43dc5bbfcb0a389948f133c24032c6073211cb1df493daac7235", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0011775, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3620123430155218, "model_name": "gemini-3.8-flash", "output_tokens": 91, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:30.207534+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "3ba71878-648c-4911-a54e-0e023d7cf108", "result_sha256": "2be978b26704546bc25cceb1383aa4fb608ad61ca60e313e92a38954b035b374", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125625, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.019477301975712, "model_name": "gemini-3.8-flash", "output_tokens": 112, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:02.997005+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "98bb299e-d414-4157-8584-4863bf4e888c", "result_sha256": "1b1b1c33d34cd352b42ef48696ee80855a02ceeb26968c215a6baaa212f845d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00112875, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3733802980277687, "model_name": "gemini-3.8-flash", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:33.379230+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "89b786d4-20a3-4908-8d12-b5629077951d", "result_sha256": "2eefe796ff0c141da78caf943664e6e45198ef3d52e7f57be3cccfcc26998a6b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0011175, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.384736948995851, "model_name": "gemini-3.8-flash", "output_tokens": 75, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:00.716426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "250082ed-83a8-4677-813c-9d96b61f5e39", "result_sha256": "8c2eb50d72f90aece9e1947020a139e018465629ae75c2e5bec4625b36970fdf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00115125, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.4303931050235406, "model_name": "gemini-3.8-flash", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:41.979998+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b6a3f37e-3147-49d3-9aaf-fb4ee851dc31", "result_sha256": "ca64beac62ae324b5dc9371d5fb9b78f9f9fc06ac0a5f30be2af0961e26e0417", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012075, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.6746932459063828, "model_name": "gemini-3.8-flash", "output_tokens": 99, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:17.824034+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d90704e8-2718-4c9a-ace8-ec3616a656d9", "result_sha256": "646ea60b7ed24d7ede640f9264ec30e21bd8c9054ab6efdcb2fae61d928114bb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0020535, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.693159295944497, "model_name": "gemini-3.8-flash", "output_tokens": 305, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:47.149115+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9dc0840-bbba-465c-9c39-ff061da9a690", "result_sha256": "6833fe77848895e2e444d416b84e86a7bbbeb883cb50071ec3cf48f4d4a7acbc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00234225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.8541606669314206, "model_name": "gemini-3.8-flash", "output_tokens": 382, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "google", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:26:33.805924+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "233c31db-199b-4f18-bb8e-e6f4ee996be5", "result_sha256": "d758d45e7f4c0c4b55ac4ab6b7144293c6f78961a62cf9c1140fe05a5e8292dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00227475, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.597528710961342, "model_name": "gemini-3.8-flash", "output_tokens": 364, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:57.844519+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0f0d3832-7f24-49ee-add2-0204ecb49ad2", "result_sha256": "20a44cba5283ba68d1ac9d5b18616f830de3a0e6a3ff70cf424de58ae2951995", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002346, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.318502958980389, "model_name": "gemini-3.8-flash", "output_tokens": 383, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:13:56.980170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc95d136-b5d8-46ba-a612-3ba4271cecf3", "result_sha256": "0f60d9f2037b190c74f495d3f7339d9aadfec1c846793c6f11b42c94bed56257", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001656, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.5664854989154264, "model_name": "gemini-3.8-flash", "output_tokens": 199, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:12:16.959825+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f842c1b3-8c54-4f03-9af7-dbcf8f1b17c2", "result_sha256": "badd0fa99f8103cf945a1e6ecf63feb71f75f295f28fc5ff4a51bee9c9111a21", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00208725, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.5887570660561323, "model_name": "gemini-3.8-flash", "output_tokens": 314, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:11:33.732115+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e05a1a51-dcb5-4b1e-9308-97e7410906f0", "result_sha256": "f27c4e5f01dca4374701aa33b681678761741a2277a9c9550b95e6763ceb7344", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00185475, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.1034698580624536, "model_name": "gemini-3.8-flash", "output_tokens": 252, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:26.756189+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "cdd285c5-3df3-495b-b825-2cbab14f6741", "result_sha256": "8d0f3f29c7ed1e5eb90528433a481f0684f6f32529c0b3a3726555d2fa5540f8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00142725, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.588262823992409, "model_name": "gemini-3.8-flash", "output_tokens": 138, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:18.122908+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "adeb9dbd-52cc-4a9e-867a-4b930bb4f9f9", "result_sha256": "2f0ae704b59652cef9da0bfe6390660b084059645f70af4f5213e9efa30a26ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0019785, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.8202079840702936, "model_name": "gemini-3.8-flash", "output_tokens": 285, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:25.655163+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f30f21fc-6dab-4d66-b84c-b8b33a052678", "result_sha256": "61b8a6fab89e75881644792067f4f79097d08a3df165521bc060ad2e0172d41e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00152475, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.7785426808986813, "model_name": "gemini-3.8-flash", "output_tokens": 164, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "google", "raw_prediction": "{\n \"choice\": \"E\"\n}", "recorded_at": "2026-09-17T03:15:23.331668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7aa2ced-a82a-44f5-a0f7-67c046a8b0ed", "result_sha256": "bec88497762b42adc152b364f6474e5e3eeb64746e1d389d481532a9828276cd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00200475, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.036297575919889, "model_name": "gemini-3.8-flash", "output_tokens": 292, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:50.687954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b542cfc0-d38f-4a92-8a4f-2c530e9f9966", "result_sha256": "bf4fe934494e600726b5a1c4630b02d0bc38348b950ae054f087f3c0b0e1a80e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00240225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.9353675961028785, "model_name": "gemini-3.8-flash", "output_tokens": 398, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:25:54.716372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "3748f5ce-576f-4948-8aa9-b74a5f47ee8c", "result_sha256": "73e0fb0d877fb2864f37496896a8391ab76ac3e6545ff525b453055e772a396c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00150225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 4.436489715008065, "model_name": "gemini-3.8-flash", "output_tokens": 158, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "google", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:26:52.897435+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b957fc7-e698-4175-a119-319722ee157f", "result_sha256": "be77c2cef5ea64d8c27f942d01d19e1593e084e954db1900ef7ebdea9fd8ad49", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00195975, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.809975539916195, "model_name": "gemini-3.8-flash", "output_tokens": 280, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:17:47.993125+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c85dca68-af86-4971-85d0-65fb5b23ae95", "result_sha256": "3c2e4153c38157ab4f7f9247002b1e80f77f5cbbe06f1792d43ea739a3e31900", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001911, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.6479076870018616, "model_name": "gemini-3.8-flash", "output_tokens": 267, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:36.544404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "626df213-ed2b-43aa-8391-ff713d37369d", "result_sha256": "b2d015ecd08b5b8889d1375bd1fd65abd073b079f72cfc14ccb3a1d21045934d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002031, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.6003174340585247, "model_name": "gemini-3.8-flash", "output_tokens": 299, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:24.904204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "347db217-9819-4351-91d5-eaf5c6101d97", "result_sha256": "485befe24646647269917c565abaa17932c00bb79eeb1819c76a086d0397b35c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00312225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.8676504539325833, "model_name": "gemini-3.8-flash", "output_tokens": 590, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:23:41.457202+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "fcbb64ba-e991-4d6c-b46f-4422741bc4ab", "result_sha256": "8feff6af9a19afcd609179ed3ac755fe2e143d71e466c603f46fc5943bc01cfb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00300225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.378972811042331, "model_name": "gemini-3.8-flash", "output_tokens": 558, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:26.635131+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "764044fd-41d9-41ed-ba15-5822c0bd5a97", "result_sha256": "4060ac239c84c7eae6fc4876047117667af87d0a090fb811d42fb7bb48fa4037", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00306975, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.458273339900188, "model_name": "gemini-3.8-flash", "output_tokens": 576, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:17:03.469193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b09eb60f-f831-489e-bc72-055458bc1918", "result_sha256": "2de7d5af17999f5b6da68fe9b939eb74e7422392aba2b8c1f637d1fb4b91ef77", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0028935, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.4326633700402454, "model_name": "gemini-3.8-flash", "output_tokens": 529, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "google", "raw_prediction": "{\n \"choice\": \"F\"\n}", "recorded_at": "2026-09-17T03:21:05.304564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e7790fa6-d019-4e50-8d62-5a686cbe29d3", "result_sha256": "c9d85e64d0ce2f283c9c11f4cc96926b558644fe8d40c51d5fce4e63cb05979c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002706, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.6345270660240203, "model_name": "gemini-3.8-flash", "output_tokens": 479, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:59.046509+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b6a39b4c-f5e7-4ef1-a534-80dbb28ce31a", "result_sha256": "f215fb5a14bda9d83f10a68c9d54102711d27c39cc039de4b1a391df64da0260", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00330225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 3.0507228209171444, "model_name": "gemini-3.8-flash", "output_tokens": 638, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:09:10.582619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "942b72a4-1719-4d1d-88b3-a19793b20162", "result_sha256": "00bcc7416ea782ff8f952e0d357b37c362ce4fbfab80aff21c29d288b617a1ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00258975, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 2.309056355035864, "model_name": "gemini-3.8-flash", "output_tokens": 448, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:13:35.602116+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d6478fd-9528-4ad0-a9a0-f066606b4fb9", "result_sha256": "2e39171cba31034cd492ddadf18409fd1707f03db6dc852f8d8ca316d45821f9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0026835, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 1213, "key_errors": null, "latency_sec": 1.9833101710537449, "model_name": "gemini-3.8-flash", "output_tokens": 473, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:55.767384+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d64ca8a1-8368-4383-9a95-dc5e2089919c", "result_sha256": "e088d08c10d765da804f4eed3a03c99458a88f9c93d942d766600e90c1a9f0a5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00205875, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.8293864410370588, "model_name": "gemini-3.8-flash", "output_tokens": 322, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:21:10.220767+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f832c6c7-ad2d-4535-878f-35ed8623a3f0", "result_sha256": "a6e95493528806087104855278ca87c29efd9ddc327f0262c02d93231dfe0730", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00241125, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 2.3341858809581026, "model_name": "gemini-3.8-flash", "output_tokens": 416, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:54.323709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2dbf3fbe-0965-425f-8382-e1116e949777", "result_sha256": "f030f72f294e2e24c445786370f5001d8fad1d1a539b2e6402446b6316e3e351", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00231375, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 3.2448490489041433, "model_name": "gemini-3.8-flash", "output_tokens": 390, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:29.711903+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "952ba353-16b3-4017-9836-b9307feb81f1", "result_sha256": "210c893fe0d345cf095ae64ed4f4a8f7ef7ee5561c88a1bdc3633bab346598da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00218625, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 3.331022046972066, "model_name": "gemini-3.8-flash", "output_tokens": 356, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:12:55.630589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "77869f0b-1ace-4cdd-b362-5420415b32b0", "result_sha256": "7ea27d4b9593e13a664cad4068d83039a0ad53c2ae4ae163bb76f211d55e23bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00199125, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 5.323709764983505, "model_name": "gemini-3.8-flash", "output_tokens": 304, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:04.223328+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2978c1c-698d-4dd5-ab91-13a1b31ba66d", "result_sha256": "1aac1c11b1e50803cf56b0791e5b043475fe20504f56e6856a9da2156eb0e0a8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001785, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 2.6107483960222453, "model_name": "gemini-3.8-flash", "output_tokens": 249, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:02.347390+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3f5d573-6dd9-471f-bbee-437eaf47a0cd", "result_sha256": "df9eb3378012798bf511b71ea62c145ed723a0d64518bea4553c75e1f9397f7e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002055, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 2.436765832011588, "model_name": "gemini-3.8-flash", "output_tokens": 321, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:20.384419+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f2ed844-521e-4b6f-93ac-cc4e5a938fee", "result_sha256": "2ecff2627e19717f1d375d49d46281b7578c6e0b94b3a22d6bbb0b659499a74d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00175875, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 1.9074553101090714, "model_name": "gemini-3.8-flash", "output_tokens": 242, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:36.307410+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "8fde3d9f-1b70-451a-bc11-df3f1093e18f", "result_sha256": "c630037366ac9d950e68fad1a107c00482dc42dbe9d615a39e3d18c2df7f6130", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00289875, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 4.966433132067323, "model_name": "gemini-3.8-flash", "output_tokens": 546, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:11.477826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a26cf410-1a51-4735-9792-1bff1f7bf875", "result_sha256": "bd25668d3b5d62245449ca4dad8f9eef2c3fb529b4102c6d89fd13b1633a26eb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00339, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 2.810856053023599, "model_name": "gemini-3.8-flash", "output_tokens": 677, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:36.075583+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "70fbe473-b84b-409f-af90-2899a5d10273", "result_sha256": "97bbab6a42875011ae197fe5901b0f15fcd4ca10347efeb358cc4f8a83ed12b3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0032325, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 4.397105638985522, "model_name": "gemini-3.8-flash", "output_tokens": 635, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:22.174507+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1011e2b-a2a9-42d2-8d05-cfa62ce93352", "result_sha256": "96a136f0b3d330be6b50ee5eeb62290197cd3442b4e9485291b25eb42e3dd02e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0032175, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 1135, "key_errors": null, "latency_sec": 2.956240388099104, "model_name": "gemini-3.8-flash", "output_tokens": 631, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:59.796296+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4633d83-b6e4-4275-babe-252758ad07c4", "result_sha256": "8e77f8569321933d822deac0792760e96a877935d08bc8ecc95cf65cfb6d6d59", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00124275, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.8006067739333957, "model_name": "gemini-3.8-flash", "output_tokens": 108, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:10.347010+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "36c6bce1-abd6-4cad-9f82-e9fe56ee5c55", "result_sha256": "53e3d6a8c00e46544c9d1da06864de66c0e72336867aa9c215278aa94379e91d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00145275, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.4497610720572993, "model_name": "gemini-3.8-flash", "output_tokens": 164, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:24.624267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ab4d46b1-8750-4582-af5e-9344feede5a6", "result_sha256": "418fb6ddb6749378180983ff72c174b3a7faccec17868fe14fe8b29deff95bde", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125775, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.2099881729809567, "model_name": "gemini-3.8-flash", "output_tokens": 112, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:10.770292+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d5e9cc69-1d01-47c9-b087-efef7a6d46e1", "result_sha256": "060ee00d4c889aded0ab75a71f0a980fa42ae645c3b6e93379c0ea9395b13dee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00144525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 2.2455130990128964, "model_name": "gemini-3.8-flash", "output_tokens": 162, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:44.017273+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ad27598d-7e7f-4ee4-bea4-ae872d225bdf", "result_sha256": "ae23089f5ee5a380d00921d5478e116b4f02fd57f82b4dabef3d283133bca2ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00123525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.0431890080217272, "model_name": "gemini-3.8-flash", "output_tokens": 106, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:47.526155+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "bb21c46c-2498-4e7c-ade9-96a9be1ff592", "result_sha256": "dc7fc1f6e4a2ac39890863c1eeb4e941d7c0b420988779be9d229e3355cddc76", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0011565, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.1346153180347756, "model_name": "gemini-3.8-flash", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:14.900961+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "94f5ff5c-b5e2-44b1-8a95-7d17af6a6a8c", "result_sha256": "dff53e3dff85b90033ff4e6b6443398a73a757c3842b3bea68f94b0a0c659ce0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122775, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.4182926309294999, "model_name": "gemini-3.8-flash", "output_tokens": 104, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:08:55.742912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "7680fb1f-ba31-4876-9db7-0fe2e8e7b69e", "result_sha256": "dc93bbac74512bfc9e943d76f40788ff8a9eac62bdfd8b1cbbdb9e23f5c4fb5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00138525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.6503496440127492, "model_name": "gemini-3.8-flash", "output_tokens": 146, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:59.981853+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "cb3d59f6-7efa-43a6-bf75-fb0dce089254", "result_sha256": "24ad9aca86223c1244eda721d681c33560cb20f08110bfbf964c797b52ab0dfd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00119775, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.2946618710411713, "model_name": "gemini-3.8-flash", "output_tokens": 96, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:38.713531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "42a96ac6-365e-404b-9c84-1e44bdec8692", "result_sha256": "665e35e4d44f42b82c990d39e8d3d99fc1c829ce08277fbd0a6b8817c371f413", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00138525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.6161276509519666, "model_name": "gemini-3.8-flash", "output_tokens": 146, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:30.595033+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "7c68d7c5-becc-499d-88d6-d9dedb34b7ee", "result_sha256": "c09d6fa11926866fb0a5878790984d08fc1699f37216fcb04b2ef497cce3e99b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00120525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.5154273069929332, "model_name": "gemini-3.8-flash", "output_tokens": 98, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:33.955495+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a5f989d3-4fb0-45db-b726-4ce5bb533216", "result_sha256": "be591f30609d2d7c7c5a62e265f62e65f146d8fa7a5f01de7f04b382b9d33219", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001389, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.2497513659764081, "model_name": "gemini-3.8-flash", "output_tokens": 147, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:21.994177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9e7fc76d-4f66-433e-a21c-53cb3b5efce6", "result_sha256": "a0323f3afb4c2676c325c4477ce04b1e1b346f37c1469e760102c1e5deb5c04e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00432825, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 3.8784820300061256, "model_name": "gemini-3.8-flash", "output_tokens": 928, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:11:06.885381+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "09b97496-cb96-4c92-98fb-edf3d77aae8e", "result_sha256": "0ac45847ff323f78c3c2a4671047acb54b3730c70e19a2edc55bb038562df2fe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003792, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 3.3399625789606944, "model_name": "gemini-3.8-flash", "output_tokens": 785, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:32.802017+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9d5af3a4-5a49-4c94-bef8-53b78751266f", "result_sha256": "0d9c828207d1121df5408d4fe937a597ea707ca1674a9ddb56cf32fb9c87b1b0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00378075, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 3.2924681280273944, "model_name": "gemini-3.8-flash", "output_tokens": 782, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:17:14.928684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f0fda2ab-544f-4523-be17-83e7c33a8377", "result_sha256": "daf6eda4744999baf3f3e9334f83385b8b94b1936e0394bb6d162c653739c53e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00409575, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 6.167217475012876, "model_name": "gemini-3.8-flash", "output_tokens": 866, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:12:34.146340+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a988e8f-25bd-4282-b0ad-1655861ddc71", "result_sha256": "35b2447e86de57e30f67dfc01bf91ff226cea9ba4b295a9c1b6d84f5c75a73ac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0034845, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 6.851408962043934, "model_name": "gemini-3.8-flash", "output_tokens": 703, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:34.925750+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a389071-04a5-4268-bc94-ded4af1e7b20", "result_sha256": "87eaa9782b12796096cd19016a8c9aad9fff581352b8006fd7b5d0cc85add865", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038295, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 3.383284442941658, "model_name": "gemini-3.8-flash", "output_tokens": 795, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:41.921669+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ef19da46-7816-4933-a3c1-3bb06c7f5219", "result_sha256": "73959d7443c2bc1e90ecd6d7ab5332ae1a93f765f2b25a950bea5fe1864efcbf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00486075, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 4.522623442928307, "model_name": "gemini-3.8-flash", "output_tokens": 1070, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:02.085169+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "3d59feb2-3ee1-45d3-a2ff-915a6983e741", "result_sha256": "243661246e183000fc54dc3dcc9aa3d1a6049054a97e420d68dce09b3fa39ea3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0062895, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 5.564051093067974, "model_name": "gemini-3.8-flash", "output_tokens": 1451, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:30.208489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f74bd936-711c-4b31-a1df-11fc1473f177", "result_sha256": "44877b4912198df05109bbefc2f8cc9da20bca42195430987b13a8e64460b134", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00591075, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 5.384695586981252, "model_name": "gemini-3.8-flash", "output_tokens": 1350, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:43.679571+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9da1ce45-345b-4867-a1af-167d61fd056b", "result_sha256": "36ad17835efd24c2a7b71fe07826f6efda6e959dc36068b87fa366cba0ffe5e2", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00505575, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 4.255918047041632, "model_name": "gemini-3.8-flash", "output_tokens": 1122, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:20.054780+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c6496a7-bf26-43f9-baeb-2138d3b41347", "result_sha256": "8abaa078f22a6af7048ea7a71204e153c9d93b8dd470a8db95e18146d5efcfcb", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.003177, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 2.6632636020658538, "model_name": "gemini-3.8-flash", "output_tokens": 621, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:18:10.855018+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "cdba931c-ec4b-49b4-b499-3101cc333d5d", "result_sha256": "48125483116d728a299097ca635ac4b67a0662333e9bd9178a152273773cea6f", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00302325, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1131, "key_errors": null, "latency_sec": 3.8924536980921403, "model_name": "gemini-3.8-flash", "output_tokens": 580, "prediction": { "choice": "B" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:23.345545+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "631c34b0-0429-4474-8382-9ed5e547eb17", "result_sha256": "18e8430b4e44862b6f20e55eabe9619e9a9e58e6d022ba67a26aaf6b7dc2c203", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012615, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.7547025040257722, "model_name": "gemini-3.8-flash", "output_tokens": 113, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:05.597081+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a4d5de41-9ec5-4519-947b-23ed227039ef", "result_sha256": "87d42e27262f900ddc68123a1a52a7da7f5c080be103bdfff6fc7c515d154a6e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00130275, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.719891680055298, "model_name": "gemini-3.8-flash", "output_tokens": 124, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:12.402533+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a7489535-d8d5-4b15-b5c8-cb541b3efd7c", "result_sha256": "19a9f1fb6c5ffd6460966b5920ac5c52adfc73b3f772f264cefcf606374d1456", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00120525, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.7218739830423146, "model_name": "gemini-3.8-flash", "output_tokens": 98, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:03.285198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b539957a-2664-4bbb-ba28-9efd9f5024e1", "result_sha256": "703497aeb3ffa1f62cc698e925e7aad7bf5d79fd05e8b6f9c893893237f9ebd4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001329, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.3216303659137338, "model_name": "gemini-3.8-flash", "output_tokens": 131, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:53.406362+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c6f27732-fb18-40f2-9783-409f28e31deb", "result_sha256": "48521a5fe6d4e69d9ed94bec3574c40f20a9669cd8b78b7a4ee8e622389ba045", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001239, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.225702491006814, "model_name": "gemini-3.8-flash", "output_tokens": 107, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:31.869020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e5eb65db-f65d-415c-9736-5e1d27c49c0b", "result_sha256": "36f516c4459b7c68a3d9299083a811bdfbdb13a5282b944e7931d88521845ec2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0013065, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.0491720760473981, "model_name": "gemini-3.8-flash", "output_tokens": 125, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:44.113627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a9ac0ad1-a767-4d7a-a411-4195fb452592", "result_sha256": "a6cc76510adbf65a3a0d0337254e79fdb04f36eab5c04995c922b7b1dde36787", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00127275, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.6010703149950132, "model_name": "gemini-3.8-flash", "output_tokens": 116, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:52.167297+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "7f4ebbfc-e912-4643-bbe2-d9fb2a35fcba", "result_sha256": "36a8ad29561e3b7f694bbe51b566b9a1d44c7a1ddf7ffa73c1ce12a8e84dc838", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00149025, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.467562102014199, "model_name": "gemini-3.8-flash", "output_tokens": 174, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:15:04.742368+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9ce999c8-e6ba-447b-9000-3cd9fab3bb40", "result_sha256": "ff03a49d35323aa90b9db261b6472b9afac538f3f3964a74992964a7d812b3d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001239, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 3.0662195440381765, "model_name": "gemini-3.8-flash", "output_tokens": 107, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:15.199877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8d6447b-29c5-4ec8-b128-5323b4c61f7f", "result_sha256": "14c70cc05430277b705d2ad7c0d5762e27f9c48dcf5f4a562867066010b2af9c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129525, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.6078935300465673, "model_name": "gemini-3.8-flash", "output_tokens": 122, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:27.340420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3f270d9-70a3-4f74-b8f5-b04e9b81477f", "result_sha256": "8fc9f61bcccde997c2bbdf936fed972b11032c51e6a2381160f349c8864e10dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001239, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.4983062079409137, "model_name": "gemini-3.8-flash", "output_tokens": 107, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:53.286024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "056f8177-c26d-45ed-809e-810e1d313a5f", "result_sha256": "e650b1487f121993d8a9d3314a8d82914201b94eb14d5a818dda72751b75fa81", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00137025, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 1117, "key_errors": null, "latency_sec": 1.1880276219453663, "model_name": "gemini-3.8-flash", "output_tokens": 142, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:51.822711+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9cd100cf-e20c-4bd6-bb09-4cdc7723bfe8", "result_sha256": "910c47241122fc38693f2ab071907e6b13424bbd7211708bd58c3689f692dd5e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00144, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.395345486002043, "model_name": "gemini-3.8-flash", "output_tokens": 156, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:55.150059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c1ea14f-a1a5-4692-a4c5-e78beb58ca02", "result_sha256": "8d053a6c6a9a5e98c78f21be6b1eed99ba86733c497db13f09fa6351f93a339e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001575, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 2.0698981600580737, "model_name": "gemini-3.8-flash", "output_tokens": 192, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:10.464839+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2a0fa5d8-883e-465f-9de9-c7abcc414882", "result_sha256": "e4016ed31c1cdca8cf79212253c2277284d021b5091013340f172e7a4d2c6fdf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.8139414549805224, "model_name": "gemini-3.8-flash", "output_tokens": 116, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:42.786309+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac6c7dda-6d78-433d-b43f-13c5d05459d8", "result_sha256": "dfa6c4e558f6cea3bbee6e6bdc23b8c95aaa716e2a81640ce2c38696b30f4476", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00139125, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.316587240085937, "model_name": "gemini-3.8-flash", "output_tokens": 143, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:27:04.445931+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "108a4c54-64e3-4668-908a-b6635d255ec0", "result_sha256": "683abefc46d89caa451afc82235e412fd3fc01952e450eb1ae71599983b42088", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00163125, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.2691215500235558, "model_name": "gemini-3.8-flash", "output_tokens": 207, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:26.529554+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d2e22a06-f842-47ff-82d3-8a4a5a43f3c2", "result_sha256": "3cd0665c7f3993562ca10486b58c49bb86796c384111305bdf0d8e8ba9357a6c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00153, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 2.7840188649715856, "model_name": "gemini-3.8-flash", "output_tokens": 180, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:20.643240+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c6b6d70-b4ea-4d6b-9b17-68fadff68c40", "result_sha256": "04c83040ff9e343297fd7b10502a26e7622c00df932e308d88baec1e72fb96e0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00146625, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.448211501003243, "model_name": "gemini-3.8-flash", "output_tokens": 163, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:36.456164+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "8cdb709a-4d1c-4bdb-a895-2103076432f2", "result_sha256": "de4da220e4881cc82acecdb166d235a46692be5c20e5ae0d3354d0a6aa2f4e1c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00157875, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.17343122290913, "model_name": "gemini-3.8-flash", "output_tokens": 193, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:20:42.697933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "7394c32c-bf8e-4b4b-81b7-da786f91478a", "result_sha256": "f574071a5bc67f97db287b2b55a3879f3f504f679c8b1a8227fa20a31c635ebe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129375, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 3.7608423399506137, "model_name": "gemini-3.8-flash", "output_tokens": 117, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:00.094192+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "17130f49-fddc-4413-8326-631f078d50d3", "result_sha256": "74c3d490872a948d0f4ab0fc59744300e9a7c803e6caa43a60fcffcc52227944", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00147, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 6.107604110962711, "model_name": "gemini-3.8-flash", "output_tokens": 164, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:10.676944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f856c2f-d6b3-42f3-9ce0-be222743dede", "result_sha256": "09bdd8dfa1b24edc647c1bbfde1754dd25d0f84a16b45a4618afab7af9980b89", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001335, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 1.2842700988985598, "model_name": "gemini-3.8-flash", "output_tokens": 128, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:15:28.260126+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "225d38e6-f91e-43d0-b0a9-193722a248e4", "result_sha256": "4ea18d7a50326aa5aadb869b472bda43514c10ec9f6a3855a46fbbc3362a5a16", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00141375, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 1140, "key_errors": null, "latency_sec": 2.630488025955856, "model_name": "gemini-3.8-flash", "output_tokens": 149, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:18.561404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "66e5ecaa-45cb-401d-9161-e9f168ad367f", "result_sha256": "498465fed4fdf905678ff2cb57d5f77c64bc384e975937f4cc11162e88efdc01", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00268275, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.419223807984963, "model_name": "gemini-3.8-flash", "output_tokens": 488, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:45.218155+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "5da91075-ced4-43be-8d6c-e742b8c8b81b", "result_sha256": "f2d9504e5b2611cb427ccd9837955ff57f6a360bfc908946aeff9e8f7970c2b4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00351525, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.874995445017703, "model_name": "gemini-3.8-flash", "output_tokens": 710, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:17.036549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a9ee5502-5246-4bbc-965e-cd47946a1206", "result_sha256": "b4556d5b44ab29e2d6f2fae98abe888f94eb783b5320f83cf938bd2a417b47ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0034515, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.9747401250060648, "model_name": "gemini-3.8-flash", "output_tokens": 693, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:17:30.826240+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b037231d-a2ab-47b5-b911-32dee82103cd", "result_sha256": "e5331c102aa2c10ccaba92911a922f17755ee60164ca7c6335bee1f7ae0bd776", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003384, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 4.446592629072256, "model_name": "gemini-3.8-flash", "output_tokens": 675, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:14.830730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "6ab5045a-e007-42ff-8121-8d898c6ffe81", "result_sha256": "89138cf07f238b8e8a439a7aff24fa2d3636354c1d887ae10ec5d12460617f3d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0017715, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.9874720129882917, "model_name": "gemini-3.8-flash", "output_tokens": 245, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:08.189850+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac4994bd-1842-4dd7-b81d-56d3ac04ae5b", "result_sha256": "de7c5a285807e9bb6af345c04ad414210f6fee490f50acfc0386b53aff24b11b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00192525, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.5246122379321605, "model_name": "gemini-3.8-flash", "output_tokens": 286, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:21.217709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "375c23be-38af-4586-80a7-6340bb980fb2", "result_sha256": "73acd6cc1fbe4bb3912f24a04d8f5352ea38423fa4563781128e22a6629774ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001914, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.714066887041554, "model_name": "gemini-3.8-flash", "output_tokens": 283, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:45.389686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "99014cb3-7f2e-4b7a-8de2-4b95c73ebba9", "result_sha256": "eae0da887a440c0330bdd64f17570e8520925a08b1ce302490c8f95fb0417f3f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00180525, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.8732471339171752, "model_name": "gemini-3.8-flash", "output_tokens": 254, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:31.074194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2eee666b-4e32-4493-baa6-e846d162b44c", "result_sha256": "8a108f46baa4479936e0ba4949a815163e1f1c329439b1bb3dee1bf3b4574656", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00178275, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.6193369969259948, "model_name": "gemini-3.8-flash", "output_tokens": 248, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:47.160757+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f7b21c2c-5263-4aa2-bc1c-37f4221350a0", "result_sha256": "152c51e6575bf23aeade6da9812b7c487e544854b6228037ed1986011497e047", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00136275, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.8703401680104434, "model_name": "gemini-3.8-flash", "output_tokens": 136, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:23.287313+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a3e8491-80d4-45f7-aa3b-c3b6cced6eb4", "result_sha256": "a0403e8df6ca6230e6f676704b4586c0f0c417f5a3c8f80e10c75c86d8d960e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00158775, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.8458366030827165, "model_name": "gemini-3.8-flash", "output_tokens": 196, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:24.892114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0ac9c1ec-bf80-42bd-8029-4d542f2a8b3a", "result_sha256": "f5daea20f990dbe8a5a0b77e64dfc80fd7c3ee1ea770c07939eea794754d6968", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00197025, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.103905054042116, "model_name": "gemini-3.8-flash", "output_tokens": 298, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:45.731822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "bab85f39-daa9-4958-a847-88458da0a421", "result_sha256": "f9a799525ecf79ba1dd20c1421c1b413a8e93a5950bd9b5e5017d7bd78945edb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00217125, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 2.4632702890085056, "model_name": "gemini-3.8-flash", "output_tokens": 348, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:25.037485+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9daefef1-2dea-40d3-81f4-e7e85fb3d339", "result_sha256": "758a28ff8fde9d2f4b988d8ce9921fbfa31c96c60976737926f6bb93087ae26b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0021525, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.8699889859417453, "model_name": "gemini-3.8-flash", "output_tokens": 343, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:05.715511+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "df4e5ad7-f3f3-4891-98f1-a44e201af39f", "result_sha256": "b15e5b1c8f3a7d855a8ded8ff922951fd3e496878248286cc423764a5851b764", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00195375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 3.2095064390450716, "model_name": "gemini-3.8-flash", "output_tokens": 290, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:25.191497+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "fe617297-6366-4aba-802d-0ec4335cf600", "result_sha256": "9d7f73384c91987e51d0db73b1b55c9a53c1abfb7ddb779f8b079c52350d797e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00195375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 1.5565504420083016, "model_name": "gemini-3.8-flash", "output_tokens": 290, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:09:39.802908+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "36eac01e-b4b7-4e2a-8512-3ca2b485335f", "result_sha256": "dec791204c1208b26dc31dfe135d6cef04258c40458627456202ffef91388c57", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00452625, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 3.576905817957595, "model_name": "gemini-3.8-flash", "output_tokens": 976, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:18:50.154843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "28823f8c-dcbd-40ef-b4a5-0dd6b987ddf4", "result_sha256": "a9fb76bd5aafa5146c6cc1376db472c2cf41dd69e4e98d85f299785c2e9159c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00478875, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 6.02493344899267, "model_name": "gemini-3.8-flash", "output_tokens": 1046, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:15.911169+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2859d394-fffc-4de7-9fe1-3b55e182b14e", "result_sha256": "4316dde764bf18cf9153bbae7eaa56bd4b5d9c145e95ffc9d6822da1e99a4338", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0033975, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 4.22948754299432, "model_name": "gemini-3.8-flash", "output_tokens": 675, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:47.959876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "3b836275-42a0-47c5-9b34-18fd827134d6", "result_sha256": "d9abf2bc8b27e892737fa34bfbf09c4ec21f1525be4dc78d8f35cb9584824698", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00463125, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 3.8937248210422695, "model_name": "gemini-3.8-flash", "output_tokens": 1004, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:24:41.483450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "229b23f3-5e32-413a-9ea4-ceefefa80a16", "result_sha256": "682229b648bf337e8c2bc1a945e725ae585c11c4a924141ddc0081b18d763db2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00325875, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 2.959285240038298, "model_name": "gemini-3.8-flash", "output_tokens": 638, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:02.494422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d47cb193-c005-4f22-9993-c1d88c876d04", "result_sha256": "126f5a4c8472ff0d46f9b5cc0401c52d5af6ba1cbc47d6bab7c66c9f63ad71d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00295125, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 2.982212887960486, "model_name": "gemini-3.8-flash", "output_tokens": 556, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:24:15.053536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d4372bcd-b05f-4e3a-97c2-f1f22e53a8b4", "result_sha256": "40c942c5dece7ecfb6d7443ec284d6a17555141b1270d092e2baa3cfef1b6230", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00310125, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 2.319943635025993, "model_name": "gemini-3.8-flash", "output_tokens": 596, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:09:18.789949+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ae0fc597-1cca-4820-b01b-a9fa97572dab", "result_sha256": "47390885447657c412630c9b91ce66dfec135e06c285990279c3a80111211ede", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00306375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 1155, "key_errors": null, "latency_sec": 2.3387164120795205, "model_name": "gemini-3.8-flash", "output_tokens": 586, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:12:37.241431+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "1044cfeb-8171-4dfd-afe4-69638024f375", "result_sha256": "2a36f163e8f36720bb1fd54ecba5a7817242ac0d46d34100ed17aa11bd76b88e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001239, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.6576161490520462, "model_name": "gemini-3.8-flash", "output_tokens": 103, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:56.839701+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8a92857-83ac-47bd-b99b-ca8d8827d0c5", "result_sha256": "a909b5e0a21343ab3f713cd11ae5fbb290dcbc52d49e41a8ce134d5f50202fb1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012465, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.2541671840008348, "model_name": "gemini-3.8-flash", "output_tokens": 105, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:01.469591+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "46a85065-0984-4621-a23e-d5316f6ca9fa", "result_sha256": "af50219de46f9646498432d3e95eb26a72032d2dcc4ccf92914373e0a6fed313", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00138525, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.4185776710510254, "model_name": "gemini-3.8-flash", "output_tokens": 142, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:22.977933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b970816-8e08-4011-8142-490c7ff43591", "result_sha256": "cc5834f7219e1b30159603660ded104532444fea6c5ad36dff4fbc3f9187b7cc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001329, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.1827767750946805, "model_name": "gemini-3.8-flash", "output_tokens": 127, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:54.869363+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "bce4f1a2-687e-456c-9122-e8196e575af7", "result_sha256": "f1dc6499359f471fe62ad0075cdac8032b297aa7acab20dfffd81c0417e43fe9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00119775, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.259893708047457, "model_name": "gemini-3.8-flash", "output_tokens": 92, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:48.779579+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5a753b1-ff16-4964-9598-cbb2735bae5d", "result_sha256": "48ee3b9e3417468b5411bb8b7b5fd284d32423742e53aea3313fec93e2741a9a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001299, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.9851440729107708, "model_name": "gemini-3.8-flash", "output_tokens": 119, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:21.728672+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff5c7c38-1052-4d44-be1f-5a2cb65674d1", "result_sha256": "2246f80496d4c605afaa1f8a5aff0895c6b64a12ca2e8f7c344a5dc48975c2cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012015, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.4198996940394863, "model_name": "gemini-3.8-flash", "output_tokens": 93, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:28.912405+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c793ed6d-8cf2-4dde-98b1-e48d08850881", "result_sha256": "5d7afa9e8357275eac79c896e666886e382fa2088a6b718cc10b23ab3814f94c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00131025, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.4418715180363506, "model_name": "gemini-3.8-flash", "output_tokens": 122, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:46.708147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ec42858b-cf2b-4722-b086-fa34bd20da04", "result_sha256": "f765fdf43b832c71abe8728c918a7c74e5f2e83e8d1ad02e4d719aca1f4389eb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00134775, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.0356479549081996, "model_name": "gemini-3.8-flash", "output_tokens": 132, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:10.332834+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "563dc782-029d-4bf7-8cc3-30d434e7fdf4", "result_sha256": "7b761de62380590d3cd6969368830d039c3568785f2d80eaf1d5caaa1b49b664", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00137025, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.6983473630389199, "model_name": "gemini-3.8-flash", "output_tokens": 138, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:14.275739+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2dde33ba-47c7-48ac-9214-1bcf5ae4faba", "result_sha256": "9bf68832e5a4a12ee23292eca681a20a4e887e62ab2a62fe726296f77b0700df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012915, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.100816699094139, "model_name": "gemini-3.8-flash", "output_tokens": 117, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:23:34.106516+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "79c42f2b-d471-4118-af95-ca7049a6bdd5", "result_sha256": "ef0d3e7ab60ebe530e974cc40c589404acfd36ae7610e032535ffb0fba28525c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001359, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.237039059982635, "model_name": "gemini-3.8-flash", "output_tokens": 135, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:13.025397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "77418d09-6ebf-44b3-84ba-d83f185eb61e", "result_sha256": "c5020f707296507212fe35ec1ff277ee057d20ee2f75aa28360b38ed7d04bc1b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122025, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.4211211979854852, "model_name": "gemini-3.8-flash", "output_tokens": 98, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:46.811115+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e1a1adf0-bacc-4b83-b2c9-fa0b87a9c0c4", "result_sha256": "9451297db090e7e4b5c66166a4cd2cd64fa64955917d7b4527ebdab64be1304f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00127275, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.4899919169256464, "model_name": "gemini-3.8-flash", "output_tokens": 112, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:34.905132+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e09575ec-f255-4013-943c-57ef3f96fa24", "result_sha256": "8e44c0c19703ca42e9804f49394d4acc3b25046a0b48ce08e34fd7ad11cde62e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00130275, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.1589400880038738, "model_name": "gemini-3.8-flash", "output_tokens": 120, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:39.391078+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "3ca4b0f6-2c92-466b-b91f-43b41ccec598", "result_sha256": "a225c23230408efc3749d6222cded6c99da9ed0eac0f184ad3adfba5bcb117bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00134025, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.7493590359808877, "model_name": "gemini-3.8-flash", "output_tokens": 130, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:34.314504+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce358d74-04bb-479c-8432-d901dbaa88a8", "result_sha256": "5d10e69906c5378a5f8255cdc24d459a997b3ea9afbb4c60eb4d316186933a95", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129525, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.506996867014095, "model_name": "gemini-3.8-flash", "output_tokens": 118, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:14.327419+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f55d077a-b213-4c3e-ac27-dd9da6808f1a", "result_sha256": "fd8a52a20872887863c457b9eea80a07243dfb0008c51b7efe2798af235f0f10", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001329, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.5243714089738205, "model_name": "gemini-3.8-flash", "output_tokens": 127, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:26:16.402919+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "188e31cf-eda2-4b6e-bed5-172ea21cb6ca", "result_sha256": "244e8458bcbedc3a285077d92ac2bad6cc68b06415ec7a44e7cd8cc4d541d37f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0013515, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 3.4884608319262043, "model_name": "gemini-3.8-flash", "output_tokens": 133, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:56.845691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d081b300-43f2-4cb7-95ab-3c93c4569289", "result_sha256": "f66acac9bf8b704afb16e91262a223acfcd87d5cde6ec22a7c6657b3d3d1403e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001239, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.2208751519210637, "model_name": "gemini-3.8-flash", "output_tokens": 103, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:47.869793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "8cdd4e7f-adab-49a7-986f-3f319f95695d", "result_sha256": "df7455b0b3616cd09be2b744b8ba37d1d3bb103314b604cf4194f8a44e1151b8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0013065, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.2896674459334463, "model_name": "gemini-3.8-flash", "output_tokens": 121, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:30.098107+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "6f836cd8-bcbf-470b-9686-f42cafa71fcc", "result_sha256": "220d797e58d495015820eb823448aa7a67fe812156b253f785d948fd25c418ff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00139275, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 2.039161429973319, "model_name": "gemini-3.8-flash", "output_tokens": 144, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:06.621814+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c5a3e18-1e33-451c-8d47-832e65e6da22", "result_sha256": "9e2214a7d1da5cc9a4849bfe83e5b000fa3ca07314104d00ce0327c3c6a264a8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00130275, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.0751137239858508, "model_name": "gemini-3.8-flash", "output_tokens": 120, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:38.771942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c08403ac-f1ba-4f47-9773-f20464082207", "result_sha256": "2a3edcdd646f4500cf61caf92f0c4e8c01a19fb08b9b9c1510f3e08be95e0a8e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00141525, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1137, "key_errors": null, "latency_sec": 1.7617136519402266, "model_name": "gemini-3.8-flash", "output_tokens": 150, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:07.584466+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "6a229927-6156-41b3-8d60-14580f2aaaab", "result_sha256": "076e838869eb68c1fa5a1c70ff5c08a4464b016cc1fc64844180913981d5143f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00191625, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.8589035790646449, "model_name": "gemini-3.8-flash", "output_tokens": 281, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:22:23.774141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "93684435-5ff6-4ac4-87f3-97960bff4f40", "result_sha256": "1557e9b7de47081cdaab6537baf3b7ee2f9eaddb9bc6d0c2d3ee9cf9eeb194b9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00160875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.3576240780530497, "model_name": "gemini-3.8-flash", "output_tokens": 199, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:26:29.888626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "553c2bcb-4f96-44e8-b8c5-8d5bd3188c23", "result_sha256": "7f46f74cbfeed9e819bd1fa583bd329e8083ee2623d24813502d3fa27a821d66", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0016725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.5393984199035913, "model_name": "gemini-3.8-flash", "output_tokens": 216, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:19:30.193052+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2184244a-53ac-4ae8-b836-90eed28dc4b5", "result_sha256": "0540bf44db8365f9b4e9099d4717304877c731b111c91fecee6dc6ef8424098b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0019575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.1251641099806875, "model_name": "gemini-3.8-flash", "output_tokens": 292, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:22.405130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "8baf65f9-cdd9-4f5c-b70e-fef70f426d07", "result_sha256": "5216a870fd7116005e22eee19492186b4f352aec6666ecb14bc254e5ac1f61b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0016125, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.8058915979927406, "model_name": "gemini-3.8-flash", "output_tokens": 200, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:09:04.949659+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e3e80ea6-71c0-4de9-8d54-a395f7b73dbf", "result_sha256": "161fe577f8651174aa4c15112155cce92fe8217df08e67986bed33908300b7a2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00148875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.4228718189988285, "model_name": "gemini-3.8-flash", "output_tokens": 167, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:04.668654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "cbef9252-69e3-41c5-997f-397d5f089461", "result_sha256": "1f2f556a237a60128d33a3e5f0b2fc74be4ae84c0db5165bbec58f00ced1e362", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00171, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.5862076879711822, "model_name": "gemini-3.8-flash", "output_tokens": 226, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:35.394238+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "909ecd82-23b9-4d19-b5c9-f98bb6d8e51b", "result_sha256": "f2e2268335c74da3eb23bff5a8165892c472cb3176dea85c46ba5aa1f7596f08", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0015975, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.7683970330981538, "model_name": "gemini-3.8-flash", "output_tokens": 196, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:25.095526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "71bb0400-5bac-4dc4-b399-17a6f3c3671f", "result_sha256": "b844d3925a709f1e79d68b9233ac492df19421fb025a20d9e9ad8db86ab2fceb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00256875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.286740691983141, "model_name": "gemini-3.8-flash", "output_tokens": 455, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:04.592808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "7ece4be4-7101-42f9-81b9-6cd4cba9372d", "result_sha256": "3c8597c6ee191a49eb50a398b9f72de69c2faf02fff10aaa3ec09c64d5a0f889", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00241875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.1170510690426454, "model_name": "gemini-3.8-flash", "output_tokens": 415, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:31.034181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "62a0c35b-66cf-4b27-9c75-3e59b326acc3", "result_sha256": "818126030d563ad0640cd12511fc5afc2fc101a91b17712a093d98cd92a80cd6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0031575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.6764772731112316, "model_name": "gemini-3.8-flash", "output_tokens": 612, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:16:20.092416+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0e7fafda-d996-45d9-b4c4-976d20cc12cf", "result_sha256": "15fcad7b52886f8f7f650ad01c20c9a65d9b17d054a2711eea0792ced1889fe3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00341625, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.435791819007136, "model_name": "gemini-3.8-flash", "output_tokens": 681, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:48.512876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ab89d42a-91d7-41cb-943c-1c8ca36d6928", "result_sha256": "bfa8f6c0e7579ea7ee92bafad2e3f57eaa511ac57a4244419f4938706ae9fc8b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0020925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.855817079078406, "model_name": "gemini-3.8-flash", "output_tokens": 328, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:19.573319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e1bf4864-b2ee-46e7-97d1-259a5590ad2f", "result_sha256": "ec2faf9c0624e6870f077e4773c7c006ce4fa999062a32b99750542eaaf5682b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00259125, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.557222079951316, "model_name": "gemini-3.8-flash", "output_tokens": 461, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:20.890644+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "37ed2969-379a-47bf-baff-0e848c644751", "result_sha256": "003d3c05a1e8bd8af99de2355bc6e4af3ce6add69e0dabbefb626b7ea78afb96", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003585, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 4.922548152972013, "model_name": "gemini-3.8-flash", "output_tokens": 726, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:40.609936+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "48b7ccb3-183e-4770-a3da-c058e2d6b5e1", "result_sha256": "e8bac0465fbc507f241edac850d57097b0eabee66f47a54663011227439cdda5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0021, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.5289983499096707, "model_name": "gemini-3.8-flash", "output_tokens": 330, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:08.792396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "94235a84-f615-450c-a202-a493bd20a85c", "result_sha256": "9924afb034d57258dfc6c949b73b50c725110f998d6edbd2c74ffb2db56a0456", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00214875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.7132873149821535, "model_name": "gemini-3.8-flash", "output_tokens": 343, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:18:26.096626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "85c27e8d-d2f1-4621-8aa3-62c4578ca289", "result_sha256": "54be024b9210a40956117161457888639a418f76579b88912898775435f32d0c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00184125, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.5397661520401016, "model_name": "gemini-3.8-flash", "output_tokens": 261, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:34.344756+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "79bc8bb1-1ec4-4e4e-bed3-16c1cac200ba", "result_sha256": "f9474d8e321fdeb9eb065e42c987a5a97baf2b924c87553d9eb1160d68381160", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001995, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.8888942629564553, "model_name": "gemini-3.8-flash", "output_tokens": 302, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:25:25.836508+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "bb0cf37c-c65a-42f8-8c09-3fad64edaf2f", "result_sha256": "b6f0a1bdd9968d01cf8e3e347ac6863c4a3f061abbc10ac25d5270c711e4a045", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00198375, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.34392602997832, "model_name": "gemini-3.8-flash", "output_tokens": 299, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:13.636223+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "44cf8bc1-8dbd-4af0-8b83-e8c8a56cc2d6", "result_sha256": "477480a12793cbe8bd76085b245f218a3a6a1105ce7ea4fb975ee1ffebb4ba04", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00258375, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.9131213759537786, "model_name": "gemini-3.8-flash", "output_tokens": 459, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:49.334578+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "56104778-e8b6-4ea9-88af-caefb30272a6", "result_sha256": "f98b877e866c2416922f5d072598796c18c8b3719704c1a86c9d1255a2031fbf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00231375, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 2.23195147595834, "model_name": "gemini-3.8-flash", "output_tokens": 387, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:00.451801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8defde0-2e73-42a1-be1e-fb49ee7fa45c", "result_sha256": "10c89d9382d5393164bf02b698040de3d7c3c742dc75138d5343adff7ac84e75", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0021375, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 3.6981255878927186, "model_name": "gemini-3.8-flash", "output_tokens": 340, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:30.884522+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4978f53c-a3de-408f-b248-3b8862a4de1e", "result_sha256": "87e4ba71bd6d6add04ffa826a467d48ce0601f8d8f6a552b727cc66fe078f5af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0019125, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 1150, "key_errors": null, "latency_sec": 1.6762777790427208, "model_name": "gemini-3.8-flash", "output_tokens": 280, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:50.432170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c360a608-b539-4b23-8a00-2ddeeef8cf49", "result_sha256": "cf8930bb95c484b218e1d66390c920e993e39bfe057a2bccc63f87d6436110aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002634, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 4.149328193976544, "model_name": "gemini-3.8-flash", "output_tokens": 470, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:21:01.681731+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1b8d982-3ec7-4fdd-bab2-3d2150371c58", "result_sha256": "375bc4a1e68b502d2d4f6bd1d1242f9bc6fd2ba349a8284c7f2f3f5c0f6bc9ac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00258525, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 2.113141109002754, "model_name": "gemini-3.8-flash", "output_tokens": 457, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:14:20.889594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "66016aed-1d83-439d-b52b-8b2c99d5a2c4", "result_sha256": "a39eca7160306bdba38a73affe8516b5d87a3a497c9ea58add7b84492034d329", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00374775, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 3.333822738029994, "model_name": "gemini-3.8-flash", "output_tokens": 767, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:14.766801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "af849334-8ba3-49f3-b145-1a28b0443f2b", "result_sha256": "d6d565352115027eb1449c6eecee83d6fc08fd667ce32a1bbc495bd774b00601", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00247275, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 2.245866143028252, "model_name": "gemini-3.8-flash", "output_tokens": 427, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:08:48.616663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f3b642f8-366d-4f0c-bf8f-b1bcdcc84f8e", "result_sha256": "73df42dd2d8c87d6345b4b260c223ab33d5eed61dffa562fd2046ac3c63e3724", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002889, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 3.088643671013415, "model_name": "gemini-3.8-flash", "output_tokens": 538, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:11:29.889360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "81567396-d60f-4b12-856d-d48e04ea1246", "result_sha256": "f628302d60311d2c85a3c5e08789c0a0eea6bdb4ad15f760ed0ac29e9f90e9ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00253275, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 2.0958253300050274, "model_name": "gemini-3.8-flash", "output_tokens": 443, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:34.034140+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "029b65de-2efe-476a-924a-2ed40b23b7cb", "result_sha256": "de61e3c269eb70b8e17fd47bc5400c7dd7cb054dc16745f4ca5f74bbadb067af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00287025, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 2.59082025103271, "model_name": "gemini-3.8-flash", "output_tokens": 533, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:16.121928+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea95177d-35d6-4439-ab09-a7ff91a9ad7f", "result_sha256": "1d08658e6ce45033098ffe0d5d8a941f1454309fa660eccdf05e07bcf01273b8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004059, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 3.2973644790472463, "model_name": "gemini-3.8-flash", "output_tokens": 850, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:16:46.680665+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0c6b60ce-e4fd-453c-8fea-bbc0a8939264", "result_sha256": "41f65a80c2f749a5b07e44055028a2bd67a9b30bc460a7cbb525674f7f27851e", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00247275, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 1.9101667359936982, "model_name": "gemini-3.8-flash", "output_tokens": 427, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:17:40.108630+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0d810f4a-2ed1-420c-ad63-3f2503868849", "result_sha256": "96384a0f937e434e2ed74d0183259ff698adda10923ac39840c37718f063014f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0109665, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 10.485313226934522, "model_name": "gemini-3.8-flash", "output_tokens": 2692, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:57.609053+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0df7299e-1a57-4d48-a36d-8c8a37746f5e", "result_sha256": "bfa29601178fc181aa9d772697fc05245f94b4e5cfa433e7208489cca1156ddf", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005454, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 4.546380468993448, "model_name": "gemini-3.8-flash", "output_tokens": 1222, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:37.324622+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce47a393-59ae-4d70-967b-9f2b6b717d05", "result_sha256": "15d4b642d16c4c8bbe77f8be974f20d565ad13e21405c405f40bf9749c897735", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0044565, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 1162, "key_errors": null, "latency_sec": 3.9627354600233957, "model_name": "gemini-3.8-flash", "output_tokens": 956, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:04.410283+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "222b5b90-f814-45d4-a2ea-73e67e2560e7", "result_sha256": "be5e85f731e8f4bc72d33f329ab10c70ae579c829f6a381f3e230bafe5a3fd9c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00219225, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 1.7054399619810283, "model_name": "gemini-3.8-flash", "output_tokens": 352, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:25:52.104449+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b50b6967-7cd5-48c1-8e66-e4b6b4ae37cd", "result_sha256": "3217782364ed08cc8e1ba977e8057056585f2aff7f75de1c8713af6327a95734", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049635, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.348170951008797, "model_name": "gemini-3.8-flash", "output_tokens": 1091, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:51.989169+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0ac4348-f3d0-417f-bde7-3748d8cb126a", "result_sha256": "89349deb95b4f7b4162869882609c0b531542a373331c936b392a9cb71f99305", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00272475, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.2493283089715987, "model_name": "gemini-3.8-flash", "output_tokens": 494, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:12:49.188681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4cca6659-41e2-4ca0-b3f2-20b71164767d", "result_sha256": "cb37be9e328bb08502ae7ae8b1d678df44ddf14c58b5ed85c5f9605433a1c34d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002556, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 3.1547752380138263, "model_name": "gemini-3.8-flash", "output_tokens": 449, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:27:17.742631+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "10d81f63-c433-4c45-b0bc-968e19dae729", "result_sha256": "66bbd89967a3c4313afbc956664d4ee3faf94d1f48ef24eed46de6df5a40c44d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002871, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.48307457996998, "model_name": "gemini-3.8-flash", "output_tokens": 533, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:23:52.920244+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f8b7c9d-84db-4e8d-95c3-d41ae11ec720", "result_sha256": "b52afeafdb5b1fe98d44f53c8591ad2d9faefafc08d2757de4b83280c5c1e028", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003786, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.9755336169619113, "model_name": "gemini-3.8-flash", "output_tokens": 777, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:14:41.028606+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea64e585-f126-4c52-9377-6a230ae0f46d", "result_sha256": "64b754858ad7bc68289456adb40bff4ac9fc3ea49224692e804c198bce7258bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003591, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.357607073965482, "model_name": "gemini-3.8-flash", "output_tokens": 725, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:18:18.816821+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7d1e655-71eb-4318-acc6-0c05c944be05", "result_sha256": "c61f5a5fc5e836bf524eba9613c84da6e3981342c4a8b9164f3fd89c2be0284d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038835, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.873310983995907, "model_name": "gemini-3.8-flash", "output_tokens": 803, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:42.184878+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "90b248aa-3a89-489a-aaf0-ffc1d3164574", "result_sha256": "5b152debeff7d818effbd722bc234c8f2c7772177a055dfe25cb82ff15721f06", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00235725, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.2525211159372702, "model_name": "gemini-3.8-flash", "output_tokens": 396, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:23.637022+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "84782144-c081-4cc6-a516-75cebd0bf5b0", "result_sha256": "c31982f0cee7067ba9991b9d2445d2c6b9da243a43544d0509b717b07da703bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00340725, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.7845325460657477, "model_name": "gemini-3.8-flash", "output_tokens": 676, "prediction": { "choice": "D" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "google", "raw_prediction": "{\n \"choice\": \"D\"\n}", "recorded_at": "2026-09-17T03:20:55.961050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "3df4a7d6-8ba1-4186-9c40-b06869c90221", "result_sha256": "357f10a2b42ad5152d27ce1572136ba9623b0e9f1408cb6350381ad76b38a126", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0037785, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.626098091946915, "model_name": "gemini-3.8-flash", "output_tokens": 775, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:13:07.375376+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f867c7c4-381b-46c0-b0d7-c27d7004c55e", "result_sha256": "0ab577e3e63ee4f10f41213821d45772f349800ea9d90208d0bc3697d61cf529", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005586, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.200185633962974, "model_name": "gemini-3.8-flash", "output_tokens": 1257, "prediction": { "choice": "A" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:32.928536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a3acbdb3-0eed-45b0-930e-37128503ecee", "result_sha256": "50902df576ed77e77a99deb21fc56cfd2b17f9656b139f65d49c4500ec3da579", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002451, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.424420356983319, "model_name": "gemini-3.8-flash", "output_tokens": 421, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:07.234043+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f86a4eac-d84b-451d-a3c7-cff3fde4e9bd", "result_sha256": "8caa94dd8a8f18b325054381859386df0dbc496b937299616632246d4c92de62", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00227475, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 2.2944600901100785, "model_name": "gemini-3.8-flash", "output_tokens": 374, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:09.402896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a180da2e-e106-42d8-b901-fcebac845208", "result_sha256": "cc953bfc102de37f51cb386d7173a1cabf11a4839eae400d20d133d51ab11bd1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004341, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 4.255485474015586, "model_name": "gemini-3.8-flash", "output_tokens": 925, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:21:42.006471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "eda1d68e-7af2-4033-8037-632f9a4c4306", "result_sha256": "b836ad91c7867848f18b894408a30ed44b43949ff9c28bdeb7b0765840e1c7d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0037935, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 1163, "key_errors": null, "latency_sec": 3.0424181410344318, "model_name": "gemini-3.8-flash", "output_tokens": 779, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:13:00.860867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "8f427057-a47f-4b03-9372-471e2723f03d", "result_sha256": "55b23352bdcbbb8ffc158b2d88d3bc1342d6798305bf5caa7125d72c095f330f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00137625, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.59395911404863, "model_name": "gemini-3.8-flash", "output_tokens": 144, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:19:53.359477+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9f6d954-2658-484a-8e4e-f40332d98022", "result_sha256": "7b8cc1b4e20f74909f741b00d635a630604c5cc17b7bfe1d659b5efeeb671408", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00118125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.5750894400989637, "model_name": "gemini-3.8-flash", "output_tokens": 92, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:32.553740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "debb5af7-2127-4c76-ae80-e80e5c477aba", "result_sha256": "af1bf59e5a3c0cb3ec9152538a13e3fdba3c049abee355a6b0ddf3da3e0b2dd8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00144, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3073555279988796, "model_name": "gemini-3.8-flash", "output_tokens": 161, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:17.985678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "8b98d97d-006a-4c99-b764-ce4323a67929", "result_sha256": "f58ca4cb11e66bc9a52cfb592c7035812d84a18082696b16994094b07434544e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0013575, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.6076133579481393, "model_name": "gemini-3.8-flash", "output_tokens": 139, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:28.192301+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b02cc74e-d675-4a01-9304-0cc96db24845", "result_sha256": "5c43b69b806ea38936fe341ccc0c102f7af982034637b019a0ab26e8440c4405", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00128625, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.4081310339970514, "model_name": "gemini-3.8-flash", "output_tokens": 120, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:27.113952+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "acee6cf3-b551-49f0-abf3-99c79e3f4b56", "result_sha256": "0e6b786aa9d948832f2fc178f4ab6072d6714c0860dc1fc9807e548d0a365815", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0013725, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.3647640459239483, "model_name": "gemini-3.8-flash", "output_tokens": 143, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:34.512412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "7ba3884f-b73a-4594-9861-32898bf21fa5", "result_sha256": "cdf2d8fa44f480c8f0efa43ab0bccdf71dc7b34c1269550ffc93d034b4a64152", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00124125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.7772817320656031, "model_name": "gemini-3.8-flash", "output_tokens": 108, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:09.222059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4875fcb8-cf6a-48bc-99ac-5ea57de43697", "result_sha256": "1ee8c37f61c28f9750ba8f508ef227a46aa55bdcca7d8666bf3f4742e65c3d4f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0016125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 12.912153468932956, "model_name": "gemini-3.8-flash", "output_tokens": 207, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:00.064791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "6c573dba-ef43-4f96-ab10-fc5ea8639616", "result_sha256": "6bcc8e6ba36e84e94c156abe25bfb82b4bef4823958a26e2412c97daca205f8c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00126, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 3.746939786011353, "model_name": "gemini-3.8-flash", "output_tokens": 113, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:03.101791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2dc84422-146a-4f41-9d60-e26209af7ba6", "result_sha256": "a22ba1c5aa5e0b8bbf159dc52f6d65231d67172062e3ac27b2dca83290d98df5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00113625, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 2.35884459502995, "model_name": "gemini-3.8-flash", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:11.441561+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "aabb7ee0-f70b-4425-81ea-00ef85e9c3ba", "result_sha256": "4e8e8fc8ebeec46d46acb4ede2233302acc389eeac82815c7bbd6036a4e05654", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00130875, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.1282771669793874, "model_name": "gemini-3.8-flash", "output_tokens": 126, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:47.649890+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "8f75fe5f-1130-4d3f-a084-1ac5c5d4535a", "result_sha256": "a8445260d982f7eca22ab3948a9a61976baa076108424c84ef3c24c990d04a21", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00130125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 1115, "key_errors": null, "latency_sec": 1.4992628738982603, "model_name": "gemini-3.8-flash", "output_tokens": 124, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:58.873968+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0042513e-d168-4bc7-820f-ad16cc2bbbfe", "result_sha256": "d7f96266d458e03a0454db4b38f25e6a070017ece41128369bc7db69e905674a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001173, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.409261084976606, "model_name": "gemini-3.8-flash", "output_tokens": 86, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:49.620463+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5d2a6b6-4b8d-45a8-b6c2-70799d58aecc", "result_sha256": "cf9a0751a6a5084819e87d9d36544f82bfc6998dc3c49543fc1b411d7d02899b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.0560475320089608, "model_name": "gemini-3.8-flash", "output_tokens": 101, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:23:14.702945+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "be956b66-4ba1-42ad-960f-7629beaba6fc", "result_sha256": "04051f7da5d4e19fd59a68dd14604b817a969d6a4f0031acb3d9b0d8e7a45b5a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012105, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.1641646820353344, "model_name": "gemini-3.8-flash", "output_tokens": 96, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:05.147434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "eae29c9f-409c-47c2-8b22-96ccd1c85b5e", "result_sha256": "db4ca47a514f9f602809676d0ce58864c1c9206f2faadf25cb2639c1fa2fc370", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.66689288103953, "model_name": "gemini-3.8-flash", "output_tokens": 107, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:13:09.158630+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "15d29632-a5e5-4973-b105-f6432bdbfd9b", "result_sha256": "8a7a1d81663dea0bb3fb44bb03f760ddf10a5d01847ac8255f54965496cbb9e2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00117675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.3043932190630585, "model_name": "gemini-3.8-flash", "output_tokens": 87, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:21.002000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "17e8f1da-5245-42dd-b956-0a3c75fedabb", "result_sha256": "ee66c0c08a0ff5122d88d8cc451dc8cf3e246b8f1fdd1ebede95d0412bdcd80e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0011505, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.2859864979982376, "model_name": "gemini-3.8-flash", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:13.191870+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4e028b33-4de6-4fed-85a0-2e933fc2f516", "result_sha256": "87395201a8b7707660e2bda5eb24ebda470e40ecc9a7eabe44d671c5ff6c00a1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00121425, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.4702460159314796, "model_name": "gemini-3.8-flash", "output_tokens": 97, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:42.690164+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9461be09-edeb-4c22-8f1f-7df464aae117", "result_sha256": "7e1b5039f5aacb7c3e5d324503bf17422bc1ec66a61df268cb57467b3abf2f81", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00123675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.9848807089729235, "model_name": "gemini-3.8-flash", "output_tokens": 103, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:52.343564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "80073f45-9ad4-4268-ab94-070c3eaa5bff", "result_sha256": "998c8a3e8eefe307a077223421002137c0ba761e52037220732e092aac2b4e30", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001278, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.513899810030125, "model_name": "gemini-3.8-flash", "output_tokens": 114, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:26.206371+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c43b3ea5-89cf-4be4-a1a3-07259561eff9", "result_sha256": "a357c707e339d576f6813b3a189255a83f3bb8dd83d9c881d0ab659d56febbd8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001188, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.1206810439471155, "model_name": "gemini-3.8-flash", "output_tokens": 90, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:27.956053+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "05a9d549-2bdb-48a8-8917-f38ba6ea5e54", "result_sha256": "0f2fb656a38727ea9da8fa2af72c05b68f3e80a09e8b3493f310964eb9c95c8a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012705, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.3400554630206898, "model_name": "gemini-3.8-flash", "output_tokens": 112, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:09:51.929705+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0b3407e-8435-4d5f-b020-9be351ca74d0", "result_sha256": "d5bb90a41c0b42b5805311c63dedf0516867bf7496cb939ca0ef8370917e7ed5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.648759257979691, "model_name": "gemini-3.8-flash", "output_tokens": 119, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:22:37.976359+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e67ce418-3d21-44d9-88eb-72ab632eb327", "result_sha256": "970a154edd8425e93724399ef512267df5363e9b051bcd7442631fe080a87a33", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.249412951990962, "model_name": "gemini-3.8-flash", "output_tokens": 99, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:10.669277+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "e9830563-91b6-4252-b76b-5dcec78ce022", "result_sha256": "0062015134d9aa2e2b57d8f0e355a198b09f834bf989e65f17e809b24e821823", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00113925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.0215410640230402, "model_name": "gemini-3.8-flash", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:36.969147+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9e65c5b0-61cc-4a29-8f48-1e83ba6b3a58", "result_sha256": "c3d120a0e3230b9ceb6240d36ebc46bd7761ca3558f391b1ef9356465bbe29d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00119925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.5502390449400991, "model_name": "gemini-3.8-flash", "output_tokens": 93, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:08.292931+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "8ff68733-cab9-4761-b503-432a17ed8a90", "result_sha256": "076083d3c05f975825020ee6d87741cd3f90e5a061351f55d78edf15c139ac06", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.3991187430219725, "model_name": "gemini-3.8-flash", "output_tokens": 107, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:19.179097+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4041a764-742d-4e15-ad95-9828f03fe4ea", "result_sha256": "b2d1a3d8c96f7c07aebd0442d242aa6d7643e2fdcf7134d225061a8e9069d428", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.3503176809754223, "model_name": "gemini-3.8-flash", "output_tokens": 99, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:18.442753+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c76b9b4-ad3d-48ac-8766-0204da17f0d8", "result_sha256": "8b2964bb2a226e7694e2424644294c15ce992ad3ccd8404e726cb5e8bcb38638", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00128175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.630324814002961, "model_name": "gemini-3.8-flash", "output_tokens": 115, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:19:44.356667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "df8f56d7-e388-4204-9562-90dac8070bea", "result_sha256": "d271a2fb58a5fcc4bccd7a5174a02e9eef057b6e17fcfd5c92163b6d9940e72f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00129675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.2729088639607653, "model_name": "gemini-3.8-flash", "output_tokens": 119, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:57.082112+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c8bfd032-c991-4658-b8e0-0dc1bccb6b21", "result_sha256": "e22aace5809540490f83e5b3f61c77a80ee9e5dec14d41a57525b61478a429b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00115425, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.457596594002098, "model_name": "gemini-3.8-flash", "output_tokens": 81, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:25:16.853832+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff3d806e-c3c1-4a5b-b3cf-2a76f5041bf8", "result_sha256": "ba3a966821b6fa9c510df854a74cd9df45f9afafe9d3357b9455be62e145eb03", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001218, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.3893025440629572, "model_name": "gemini-3.8-flash", "output_tokens": 98, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:21.837706+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a6c2dc41-2b4b-47a7-8186-b023531064e9", "result_sha256": "67ff78e4626b6581b527b9b89c7d04f1a4b87611d920455720dcec1632328a0a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00128175, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.206258254009299, "model_name": "gemini-3.8-flash", "output_tokens": 115, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:37.446316+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c180e4ac-81b6-474c-8e3f-64a2e0fe3270", "result_sha256": "2945336bd8b1e8008a1e4303889c3ba2893dc0e5fdb98ef75febbdbd785b1e01", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0012105, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 1.653496494051069, "model_name": "gemini-3.8-flash", "output_tokens": 96, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:26.874164+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a7d849e9-79c6-4e02-bcc3-8150645d7162", "result_sha256": "1ddd23f18d2280518b8f1870c416ac4ba38f3e9d85aac98ae1ff8784370ff569", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0011505, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 1134, "key_errors": null, "latency_sec": 2.1516041950089857, "model_name": "gemini-3.8-flash", "output_tokens": 80, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:11:25.418062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "22c00225-b6dc-473f-ab95-9c61409e8b31", "result_sha256": "a3636c1f54fe788a24a8e6fba2486defc343e09bfdd654bf18a4159c77071f88", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00150525, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.3144246429437771, "model_name": "gemini-3.8-flash", "output_tokens": 176, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:23:37.026134+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "dae09e77-0cc8-4faf-bb3a-87cbbe09e702", "result_sha256": "ac6b7b82eeed58c28bcb86b6b85cc62ef01e3ae75531054525917d5c39191448", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001494, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 2.138473421917297, "model_name": "gemini-3.8-flash", "output_tokens": 173, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:43.547873+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "f200f730-3c13-4894-867c-c6e878674218", "result_sha256": "2e7a3c1e9ae7d7e02afe38278e0901ee1d816587134431a6bc97a30950669f22", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0015165, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.2596789749804884, "model_name": "gemini-3.8-flash", "output_tokens": 179, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:02.024802+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a881695f-2274-4b6f-8514-61c0247b85fe", "result_sha256": "b7dec845e2c0c2a8fc8e0976420e1bab3c78db02729e1752082b65ee0637d45f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0014865, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 3.1711733939591795, "model_name": "gemini-3.8-flash", "output_tokens": 171, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:24:06.594084+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "befdff7d-9f44-44f3-9516-0fb5e9ea67b0", "result_sha256": "2297314f46736c803257219f8a00d321a10484d0aa4bb4b15a4c97577d84f814", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00164775, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.4833136819070205, "model_name": "gemini-3.8-flash", "output_tokens": 214, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:21:15.455656+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d021020f-f312-453f-aaca-8350c011a18f", "result_sha256": "1adbb7952bb158f9c0caf95faaea1fce880463097c2eb7f05ee89a9ba7f1da0f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0014865, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.4603587160818279, "model_name": "gemini-3.8-flash", "output_tokens": 171, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:27:21.998549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b2818c4-4882-4d2b-9a26-d3181fe25b89", "result_sha256": "b7eb42024c7f081ad330efd1a201fb28db984401cedb84ec826a8f69be35d1d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00146775, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.5656305470038205, "model_name": "gemini-3.8-flash", "output_tokens": 166, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:03.598747+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "40d089e1-2b3f-404f-b365-aa4fcb139b0f", "result_sha256": "e9460f13018fe8de3af6bf6e94c8fb267f3d1dc07a2a92c4ce60024528586e9c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00150525, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 2.258630452910438, "model_name": "gemini-3.8-flash", "output_tokens": 176, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:19.521500+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "7bd0f244-d7e4-4d71-a512-ecfa1c4a047e", "result_sha256": "52e5e6dadb088790b813324974810b40b6fb90e4ff1b0efddce3ca98662b46f8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0024015, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 1.9631278020096943, "model_name": "gemini-3.8-flash", "output_tokens": 415, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:20:00.275386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "49b19a50-a236-476f-bb53-6a156656c217", "result_sha256": "8f009bfa096bc6107245ed68e973fd2dc947fc99cfc8846855fd2ea2b7efab14", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0027615, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 2.994213673984632, "model_name": "gemini-3.8-flash", "output_tokens": 511, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:37.553060+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "13ce1ca5-1f0d-43e7-b595-9cbcd6af130d", "result_sha256": "b23734eb52811adb53785c7a021e829ac0d62fc4e1e673045086bf0b31a9b67b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00289275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 2.7956757760839537, "model_name": "gemini-3.8-flash", "output_tokens": 546, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:20:45.802460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "bfe79d68-7fea-479f-83fc-b7a3be03e359", "result_sha256": "7f4a55e5ac06c6807d959faa059bfeb9a825d5fb8f36bf9849b7c05d3a1bd001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0026265, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 1127, "key_errors": null, "latency_sec": 2.827945190947503, "model_name": "gemini-3.8-flash", "output_tokens": 475, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:10:17.370915+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "99551957-47fb-4ece-96cf-5c7c74e5cff8", "result_sha256": "a3605ee718a037c709b6e614c989b426a839543274b88275d4337e6347abef2f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001662, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.1478824509540573, "model_name": "gemini-3.8-flash", "output_tokens": 216, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:59.521195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "14b155e1-4d11-4fc8-9a11-600fd10bfa37", "result_sha256": "ab3b7ba792aa27ea2e26a71ce3e0e215b491ef66cf0d3e36e604405698d25264", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00175575, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.5835727190133184, "model_name": "gemini-3.8-flash", "output_tokens": 241, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:34.163211+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9cbbd3dc-4aed-4de2-a520-7c2e4ac09aa6", "result_sha256": "336cb9217d33069fca57f287ede8e9496b96cba46cca822e7c1c9d93e22b536f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00222075, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.5689682610100135, "model_name": "gemini-3.8-flash", "output_tokens": 365, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:01.058025+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0b3c1f38-5505-4641-abed-e4ab7087fd59", "result_sha256": "1b1e0447fc45e3ad9c543e90406317f1036f9af54bfcbb3d09d807a74c476962", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00238575, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.405517528997734, "model_name": "gemini-3.8-flash", "output_tokens": 409, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:15:36.818931+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9d139100-47be-4758-9926-0c4a6da3b683", "result_sha256": "011b7cc20892b3bac1b607b1d6a7975a77507af28c3d083d9b7f1a51332d02a0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0018495, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.6918364909943193, "model_name": "gemini-3.8-flash", "output_tokens": 266, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:45.360614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "6c05c40d-21a9-458c-9f85-fe80de5df438", "result_sha256": "c53529046768298d2bc77c5c9e3589b289d6f23f46c9a91f525e9b341d14995a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00195825, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.6052539580268785, "model_name": "gemini-3.8-flash", "output_tokens": 295, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:12:41.881969+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c5e2258a-4362-4db2-bb04-347c372bc534", "result_sha256": "51786af872bc9068507d46e179e68f7bb1b1259872abac88cbd0b22390db2fef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00223575, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.4124497760785744, "model_name": "gemini-3.8-flash", "output_tokens": 369, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:54.583323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "a59a45e1-c489-4360-b843-bf6233ff837e", "result_sha256": "4e8767fa2ef4344292e34549d0342b9bdfb54a89f20fcb0aa12df81f643decda", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00210075, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.09246955497656, "model_name": "gemini-3.8-flash", "output_tokens": 333, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:36.883986+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "61d60605-fdac-4805-b29d-bbf20be1a2de", "result_sha256": "fda8eefcf0d589f9ce809564f6ca80f830d8386b4ce770f8a1781a6f9a2c8b65", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0014595, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.5725890630856156, "model_name": "gemini-3.8-flash", "output_tokens": 162, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:09.110684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "24d6328c-0db5-426b-b9ff-19d09faac6f9", "result_sha256": "13f3ff7b3c41de184f4d6d352119beb61b482656469bf72d1eea38bd33476e67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001572, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 1.464345145970583, "model_name": "gemini-3.8-flash", "output_tokens": 192, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:56.977525+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "c33c0559-22cb-4bab-ae6d-731f606f2a1e", "result_sha256": "8cbd66f000ab5de9a4a14fe726b2d9899f423f483c58f1ddf2fe40f4f7d46191", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.002142, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.230731250019744, "model_name": "gemini-3.8-flash", "output_tokens": 344, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "google", "raw_prediction": "{\n \"choice\": \"C\"\n}", "recorded_at": "2026-09-17T03:14:43.966227+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "da8fe6c1-c03e-4f1f-8a70-68b2a8b97ce5", "result_sha256": "37cac599c7b79579f3eac9680d23e4a7400a768033a35aedc518e358eba3e244", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00254325, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 1136, "key_errors": null, "latency_sec": 2.715864907950163, "model_name": "gemini-3.8-flash", "output_tokens": 451, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:26:54.447002+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "d6b1b39d-baf8-4567-a8ae-6af058edb79e", "result_sha256": "941d8229a54cf025e281292b0e5d603b704c57c692241e6e8a8df08c18216998", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.5172522939974442, "model_name": "gemini-3.8-flash", "output_tokens": 105, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:06.938971+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4215b35e-38a2-44da-a83d-75169a5ddb5f", "result_sha256": "ba5be17124924dca27288ca3b1d641f3fb02d2aaddbc4bd9b1821cda832f82e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.18787811009679, "model_name": "gemini-3.8-flash", "output_tokens": 105, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:32.960792+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d15f7cd-c83d-4072-b485-a515b524b9e2", "result_sha256": "184254b8b6aba593c1c8ccd80acf140b68b2a7c694bdc0127aae17bc5cc0efcb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001293, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 2.141020640032366, "model_name": "gemini-3.8-flash", "output_tokens": 122, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:51.756820+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "bf81282e-3691-4740-bd91-d440a17bf83e", "result_sha256": "9720c4edce23900d2520ea494ba0cf008cc3c665f9415ff48271b9087e1c2ccd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.001323, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.4345576650230214, "model_name": "gemini-3.8-flash", "output_tokens": 130, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:11.632391+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "5442af2a-7e2b-4e11-a9db-3abbbac2a2fe", "result_sha256": "613d85ecd93513d25789c03b5ec3b0823ae17512105721cca8c1b04b59c38a95", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.244947528000921, "model_name": "gemini-3.8-flash", "output_tokens": 113, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:04.376572+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d455ced-6fbd-4fa2-93f1-c065698ae792", "result_sha256": "20bc10efe68866c29e0cba9834e5fe95afd9c9d392f7ceeaeac5f92b98ab5a9e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00122925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.4864134290255606, "model_name": "gemini-3.8-flash", "output_tokens": 105, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:51.612298+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "9be0dce2-4f8f-4772-b8e1-e3a797621239", "result_sha256": "54b800f993f988cc68078864efbf7a160faa450b2b46357a8ae2bfa404f0ae96", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00137925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 2.7004978199256584, "model_name": "gemini-3.8-flash", "output_tokens": 145, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\n \"choice\": \"B\"\n}", "recorded_at": "2026-09-17T03:10:20.654080+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "0eb67862-e04c-4166-9336-4664a1dbe32f", "result_sha256": "29c1af281d957d4aa35dd2165260ee9290b47bc98cb544db497e7a2b886ac6c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125175, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.4500604700297117, "model_name": "gemini-3.8-flash", "output_tokens": 111, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "google", "raw_prediction": "{\n \"choice\": \"A\"\n}", "recorded_at": "2026-09-17T03:10:07.009259+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "822a3059-2c10-4ae6-a51c-dff810554e3b", "result_sha256": "8e84e670880476866642440085a12cf0b621f6c1e5fbc5e59d042081a5877768", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00119175, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 2.6007460989058018, "model_name": "gemini-3.8-flash", "output_tokens": 95, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:51.279719+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "439c2b86-d9e3-4925-82e8-0c59fe7eaadd", "result_sha256": "d9ce5570d73261dbe6b3d26ef921719d36f868f3243473b243731e7f424a0448", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00124425, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.9255729339784011, "model_name": "gemini-3.8-flash", "output_tokens": 109, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:32.219877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "df98f134-83ef-4280-b55d-cdf76c4fa2ef", "result_sha256": "1d02d4aefd3421bf35b9189d010710704a03be0f480690ece4582dc8124a4940", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00125175, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.4379473999142647, "model_name": "gemini-3.8-flash", "output_tokens": 111, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "google", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:55.943453+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gemini-3.8-flash", "status": "final", "included_in_comparison": true, "source_attempt_id": "09a27120-5916-4807-9141-14b9d66bb4e8", "result_sha256": "1aeae27ad5b9b8d50d2f135c785e1a5e91aea51a4e2b4e117817b3480a4c26ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00131175, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 1114, "key_errors": null, "latency_sec": 1.5039367449935526, "model_name": "gemini-3.8-flash", "output_tokens": 127, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "google", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:44.328157+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "6ad4afcf-6e7c-4c6c-9fb0-7583047cf263", "result_sha256": "cdda277bf252ac0e4dd1aa4620a94731cc0b5dc5c7cb1f787d744034cb979bcd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.0285452660173178, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:48.269278+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "44a65303-b02e-484f-91a5-0279842776c7", "result_sha256": "d1ee2812df29434bec1cc5a94e48f1ea0ede70164e2dbbf601d242c999231c8a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.4576550150522962, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:15.481189+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "68f0913f-9d9d-48e8-816b-219ca59edbf9", "result_sha256": "5ac6dcd1e526d80ab0b8866b52e7f1bec019b810f35500f49c4c4f2897fc5809", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.8597230889135972, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:09.516387+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d2e4aac0-bede-4881-87b6-ba0f02256038", "result_sha256": "12e2dcaab161592711550fee3f50894e1cea3c3182b357bbbee03cb3fc00b60b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.7487130539957434, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:12.036528+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "2109c5ec-59e5-400c-9d1c-a3ebff17925c", "result_sha256": "b9e1d8610326ea7e34267db58b28475a45666b4373dd6d8aa17e9d688b34a5bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.9880034669768065, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:38.562101+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "4782dcda-a926-4858-b0fd-b4d53c0ab1c4", "result_sha256": "8de1298fe7a2ffca367fc5c7143bad1662d0fd77c03699a0253677cebf60e0d4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.9490162800066173, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:53.364740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "2f8ff4ef-f3a7-4ff4-8ecf-e076224aaf8a", "result_sha256": "2add16dcabb1614119c359d394f9002a717194dffed6307e5c52536ee56cd328", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.8545646149432287, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:38.930628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "00c9c990-d51c-43e4-87f6-6a64154cd3c7", "result_sha256": "70504a2a43a20939855fa5e0be87cc8ff9510330f517ad67e25e7f1413612ed2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.8601522049866617, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:57.207170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "db2285d2-729e-4ce9-87ed-ea6c5bc7f3b1", "result_sha256": "43e74c644d74ec16a4a0a99a658a27d8d84233dec4db7fed20053486acddda61", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.9535656590014696, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:29.436732+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d2dd6d55-1bc4-46ea-8519-17d5d42d8092", "result_sha256": "c7d3b9d90f8c011b612c52018ecddb3ec5d7914f4f097b859dc8206946cb648e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 0.9023400950245559, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:51.655627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "fb723871-ae1c-4bfc-a8ca-62b7016c1b3d", "result_sha256": "ee084ff30b280337eded9ffb667df821b4e99505eefef42d10b5646e503408d7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.0543499920750037, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:29.895252+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d527972-e905-46ce-b415-2b0d9361ae62", "result_sha256": "9a4ef813d5f5a8e0a8de9f6b54b72ca3d9202aef28ff1ae96842d366fbd5e0b2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000496, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.7750284330686554, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:51.538144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d4fcfac-c2e4-4efe-8e34-be415deb2815", "result_sha256": "7c2cae649f5a33cc9b4ce43fa99a436b329feb5ad1c381b724bff1f1fc724cd1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005766, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.6917035200167447, "model_name": "gpt-5.6-luna", "output_tokens": 70, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:56.072406+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "7493aeb9-ff55-4be5-8444-b35b42ce9b0e", "result_sha256": "a2840f7fa80b92682d9d4d36369478dba631afc1831ce708c643ce15b9d8c144", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005526, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1972261030459777, "model_name": "gpt-5.6-luna", "output_tokens": 50, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:00.946998+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "61d236ee-55a8-48cc-8eb3-c625b8254791", "result_sha256": "fde0111189335a8e6aff0f212eed887b564fb8f4ff91ecb174e40b186669100f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000573, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.55294010497164, "model_name": "gpt-5.6-luna", "output_tokens": 67, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:01.798279+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "3d0a7caa-9ef5-47b3-83e4-706f673e1293", "result_sha256": "5988409a97bcfe6e195614dd873710bd8d65f4581e6d9a9f0358c6cc26eb7937", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005658000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.328513108077459, "model_name": "gpt-5.6-luna", "output_tokens": 61, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:06.841348+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "9be819fd-4e80-4e1d-9e51-5ec87a998391", "result_sha256": "364088d12ae648c6cd23cd898f7ec449fe59009c9315714fdd396855a8c0e465", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006018000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.6884302790276706, "model_name": "gpt-5.6-luna", "output_tokens": 91, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:10.962864+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a899cc1b-b2b0-4f3c-b183-05559c9577f9", "result_sha256": "411203b01920f8120bafaf9422daa6e6f51bfc53d1f3d29cc609b85d79c62c4d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005946, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 3.392854274949059, "model_name": "gpt-5.6-luna", "output_tokens": 85, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:13.461373+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c49dbb9a-1f08-4faf-9e19-3bc7e7afdedc", "result_sha256": "7ee0ce997c317651916ea0b714f347e61cceb3fbb6d4e8dcd20b83a5ba0842d4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005862, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.1617020110134035, "model_name": "gpt-5.6-luna", "output_tokens": 78, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:43.213962+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "681a6f6e-3d19-4dc9-afb3-09ce1779b9c8", "result_sha256": "88acccdb29d191bbe2e2aac8032f4b30b2b2cfb731095a39213c728fadf1031b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005466000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.6278058440657333, "model_name": "gpt-5.6-luna", "output_tokens": 45, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:42.453619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8642b4fc-debc-47fa-9cbd-acb6dc898ddf", "result_sha256": "3bca2877f7e34b117907ccf6ce654b781fc10ba1cecb61f81956606ba60ad03f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005094, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.867822211002931, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:26.905609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0c1b76ca-d14f-40c2-afd3-7d7e68800a96", "result_sha256": "4a2e3acbf495e55d1c7f1b960f03969ae2112daecc0863f09e7c0a84ef9e6344", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005742000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.4351794109679759, "model_name": "gpt-5.6-luna", "output_tokens": 68, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:21.982099+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "4481afeb-1bcd-4e00-b049-b790ab19a4b2", "result_sha256": "26294e8978472e743b1a18d739a23f594e88a1f7d780d13c0f9552580f075f67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005466000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.512335438048467, "model_name": "gpt-5.6-luna", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:07.137080+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "9fa5bd05-a1be-4430-bfd1-cde30b6d8e65", "result_sha256": "0c7bb6c6539a7e65d95957a2ba16b42ff3ba591aac2a753ad10df0d22adeb9b5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005946, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 4.127287363982759, "model_name": "gpt-5.6-luna", "output_tokens": 85, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:35.871863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d1bc562-a94a-40b1-9d8d-2d6baa69e1b1", "result_sha256": "ece55566b924a258492c270f4351976c08bf0882e1e48aba2877426c19f98ebe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005754, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.942950607975945, "model_name": "gpt-5.6-luna", "output_tokens": 69, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:51.469272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5f146da-6547-4ebe-b89b-4a0d0c9a0350", "result_sha256": "dd8f5726f89e5e5661a8b10161ff378f3dbe250208bc925190d2a6e6eec9852b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005633999999999999, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.4582583799492568, "model_name": "gpt-5.6-luna", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:03.516548+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "27e1d6fe-4e1c-4a87-906c-9ca463a5eedc", "result_sha256": "b2d398d169ecd42e6dc8c7d7957c2b5df80723a02f2cbbffcc864b3df5a3c5a4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005718, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.507989592035301, "model_name": "gpt-5.6-luna", "output_tokens": 66, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:57.160303+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "df911d68-7bd8-4477-95e7-ae70d95209ae", "result_sha256": "e9e5fe2ac796749674f2d59d72a86318bd7587f57335c34e7c2120c2719dffe7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005838000000000001, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.5936390429269522, "model_name": "gpt-5.6-luna", "output_tokens": 76, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:17.054582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "3a588645-a3dc-4577-abfb-5add907673e1", "result_sha256": "157841add074aa2c6ca1d9fcdf22b6e58b9ad9ce41cfea4cdaf7f92c8796b6c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 8.174780450062826, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:46.597488+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0d59c09-e1ab-487f-b556-92547c3b44e1", "result_sha256": "ca05bb47c7b36c2cd7440fb78afc898c3287640ff997d510915409f0332a9a1f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.8121916110394523, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:10.283843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "662dc15d-b4e2-431d-a510-8732f0a79478", "result_sha256": "fa3de98bfb695871f1848e477902c4e4729e4ea23f6c43ba2e25fef6d1759e30", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.9374549520434812, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:29.130763+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "541f4727-b695-4040-a85f-ac6c64370b01", "result_sha256": "99346f4520354b32e644f55817d89c89aa75a490aba831be7e17a5963cbe7c55", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.897829961962998, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:02.536414+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "843b0279-2cbf-4f98-86d7-ae6037c5d4bd", "result_sha256": "89488152feba95d4f5d24b7a5df3c484605e7091cc7919423b231475d13a6ca8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.7371187779353932, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:32.091852+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3308c14-776a-421e-9a22-d9355af42226", "result_sha256": "80abf544101737737f9e71f14a63f4c78d9020542e5cd0e6250deda230fe63ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.6673698079539463, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:00.444986+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "40880a98-0c69-4ebf-9f15-d44756173b25", "result_sha256": "c111468a109d3ca125486d5d5e49dd218c58a6c9d0223e6057da17981dcff62d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.9579902840778232, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:41.763356+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f6a69760-e1b5-4e05-a1dc-3de80f461374", "result_sha256": "97dbb2992523b9163e059a0c700502fd3137555a468d6a3b7a044a6d57486217", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.2334497509291396, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:17.635852+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d347e237-2232-4d2f-adb5-db3f53234967", "result_sha256": "6dddd2f1e03325885574601c1239798906a5a3f5f303d791a696370a0530e627", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000594, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.2220533039653674, "model_name": "gpt-5.6-luna", "output_tokens": 84, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:45.789567+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0875273c-74d8-47d1-91d7-7e01f327b180", "result_sha256": "007bcc3cecf43a689c731a7eca198b40a5b3c3430479d02e9b7c7fcd2ec9af87", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006636, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.989121827995405, "model_name": "gpt-5.6-luna", "output_tokens": 142, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:35.242592+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "47e97f67-e5c9-4ef6-8603-138366d3bd51", "result_sha256": "a05af6ad4f77e9713c799e7d2ca42d4f8993c52bdd508a838d31a713e6164006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006036, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.6269402969628572, "model_name": "gpt-5.6-luna", "output_tokens": 92, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:56.232831+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "7e881149-935d-4e37-8e01-c80bf636c333", "result_sha256": "bd125eacb9c92f10a11cf4c453904b387a84d525a732b2cffa2061c67e06df6d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006192000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.9440258300164714, "model_name": "gpt-5.6-luna", "output_tokens": 105, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:13:57.050096+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "de9f7a45-b4af-44de-b1f6-31a13690d598", "result_sha256": "7e466b17b0ad5b4f58c09d1369c976a08f8b2feb59384818d3d362ffb7b3bd67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005892, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.194904412026517, "model_name": "gpt-5.6-luna", "output_tokens": 80, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:17.863539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e077bea-4e73-413e-a5bb-f0d70aa86aab", "result_sha256": "d21c880b7d81bba4d9996e716b3a33f5bc4db69b23930bf7929e9f8a945a93c7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006408000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.0603284869575873, "model_name": "gpt-5.6-luna", "output_tokens": 123, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:33.512133+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8dd1e161-309a-4d93-8f36-c0fb7d438557", "result_sha256": "06a26b9d839979fed1244be34d14082c15da890511ba97d8fde836df0fc55ce4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006012000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.5082583329640329, "model_name": "gpt-5.6-luna", "output_tokens": 90, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:25.727456+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8845ade-c4b7-482a-b72e-24d3a05d6e84", "result_sha256": "5cedec7b8117b397a12e8d71da7644e15693edf19dc094248cf6f57c20b07ba0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.107137589016929, "model_name": "gpt-5.6-luna", "output_tokens": 89, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:19.204539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "b23dcdf7-f188-4b53-9bd8-f8f6790669f8", "result_sha256": "e13b308329b3e8718e9d5831e14698891129b3479c59edef0734b98bec94a8b2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000594, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.8817143809283152, "model_name": "gpt-5.6-luna", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:23.818895+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c47902c0-e997-4fd4-85ed-da5c008a2426", "result_sha256": "44a1e8a73ec2b1fbe021c3684479d3800004f9da6fe4344b16b4aba02918fe07", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006192000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.3755290070548654, "model_name": "gpt-5.6-luna", "output_tokens": 105, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:23.764988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e77fc83-1b2e-438a-976f-4a6ccfa0bc01", "result_sha256": "983f353d1a3d3a351eecc1bf2bcbd2f72bcafd31e9636f6ebac1f22715f90aa6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000618, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.1086905639385805, "model_name": "gpt-5.6-luna", "output_tokens": 104, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:51.018509+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a3172c7-a767-4a86-a559-cd1c96b65ca7", "result_sha256": "2e3164b7bee08de6537ed4007743038e4c05816b0356536c2159c05e6452e009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000606, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 5.589019121020101, "model_name": "gpt-5.6-luna", "output_tokens": 94, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:25:57.713114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0b21d260-f41a-450f-bbe2-d845b9b5f776", "result_sha256": "a4b599f41b47cde83ca42bcf9382fb01e1b72a821f09a4ec09e574f443632ab3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006384000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.8707513430854306, "model_name": "gpt-5.6-luna", "output_tokens": 121, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:26:50.011808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "36a66667-27ee-48b5-97fe-1a22b64e4167", "result_sha256": "c0df5437fa068f6691e38adb738478915c96b706c454f718e645b2020cd0b178", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006504000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.459346337011084, "model_name": "gpt-5.6-luna", "output_tokens": 131, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:47.723449+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "27e54b27-330e-42bb-998a-7220293e8a18", "result_sha256": "c229ad2d3965d29edfe3205ac197422ffbb2873f73c8010374ce11fce85e531b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006108000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.31092414597515, "model_name": "gpt-5.6-luna", "output_tokens": 98, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:38.292130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a8d295c0-0126-42b7-a93f-4bd31847b949", "result_sha256": "ac61d43b873aa39e4b47c6ebdd4453ee069c6d92fac3e9f5c209ebad74135600", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006192000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.404680470936, "model_name": "gpt-5.6-luna", "output_tokens": 105, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:25.645396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c4d798ab-e156-4961-8a99-ca9435002d76", "result_sha256": "5dc3940f25258c2108dff1263252a7fec295d9b02608ccc2fb94ee1fa86ecbb4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005952, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7491669830633327, "model_name": "gpt-5.6-luna", "output_tokens": 85, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:40.107725+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "2fb2dcca-ef48-4015-a929-4f10a38f13ef", "result_sha256": "b0b5f0669fcc0f82ed1d730dfc21258b8fdb8fa9e8f2a90b84c01d071d041344", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005916000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.9521894060308114, "model_name": "gpt-5.6-luna", "output_tokens": 82, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:26.024326+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa435b29-22de-4fc5-8006-1aeed601fc73", "result_sha256": "a9bd2fcc82235851cf6d8f392c793b38c0829e1e1715b03295121d3ccad906ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005916000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.4661026389803737, "model_name": "gpt-5.6-luna", "output_tokens": 82, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:02.113411+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1dabd492-c478-437e-8f99-bce0334b2c24", "result_sha256": "666a46a6b4835c9714204dc73e75ad4a98e3b2161e974b4c2be68a21bdf173de", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005964000000000001, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.3407087059458718, "model_name": "gpt-5.6-luna", "output_tokens": 86, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:05.461018+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a91c88d7-99e4-41c6-9278-d76c3827103d", "result_sha256": "46c6b2cdf1e6d0b4ec9f275d5fcace3b7a5b85ea73ea66cb6cc4de02e50f4db8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000618, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7791024670004845, "model_name": "gpt-5.6-luna", "output_tokens": 104, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:57.664333+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c4012699-df49-4b6c-a3d4-136188a6c358", "result_sha256": "07f0f10d12f3d58fbc0b07ac54377def6aad452b43a0db170ee8e6f0c070b148", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006216, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.6831432330654934, "model_name": "gpt-5.6-luna", "output_tokens": 107, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:08.691793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "6396c24a-522f-4efd-a14d-7cc31cfc23ef", "result_sha256": "23338ea0144a72dc8e3182514ada354482643fe4a4d65d09e9a2fde76b44666a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005904, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.8220316580263898, "model_name": "gpt-5.6-luna", "output_tokens": 81, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:34.420882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d068146b-e65a-426f-85f3-de8c8cdd45d9", "result_sha256": "636768d78b2c09a4e1b8713bef2abff889996527c6b650390820fa40fac9edb9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006036, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.631417185999453, "model_name": "gpt-5.6-luna", "output_tokens": 92, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:53.923259+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac07e74a-5c5b-4415-9815-da9ddab789d4", "result_sha256": "a777426ab7090ff32654ae95b309a298fcb2c1401af59f89f11b140b441005db", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.6963416149374098, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:10.340375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "3817f86b-7ac8-4ca5-83ab-d18d3fe3c71f", "result_sha256": "3dc7b70c8d5b21d59be7630600468be4ceb0341e6e75a40c8b1d65d660b68b05", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 0.9333124560071155, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:52.601001+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f22b5dfb-a615-4eed-8914-b29f470b21c3", "result_sha256": "e01abec901b11bba89fcd93a490e17d977707ea9fc1a6b94dad5632e71544ce0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.0804049379657954, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:26.768752+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "6ea261ab-fde7-4df1-9d51-9553873a7de7", "result_sha256": "d91245ab55e0898af49a0d702cf1fdc6071852839021b1ac0b807b09ab854f2d", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 0.7689648360246792, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:52.252837+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "3da9a77b-b254-4b92-b1ef-97b79d27e7d3", "result_sha256": "c62e8800eda1b2f87940bf7dacdbc132e56f9052ada8c1bcfe79349960a901b9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 5.2820413479348645, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:04.031897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8f64e731-5286-4b8b-acf1-ee3b4f096d4a", "result_sha256": "6dfc613f1ec642272d03d0464b9b2c3ded78d98cf2f8f604f04e24648d99b0cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.0990428420482203, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:00.001367+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "e99cf100-3947-48c6-8244-595eaaa27178", "result_sha256": "ca611b1ce6840068ea541aa871a82b9da9639f5e94bb110cf0ea10995b83f97a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.7791079480666667, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:19.086563+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "548f4c81-ce86-4d97-81e9-c7e093502edd", "result_sha256": "803c4bcb8bdf31141769c1acc560e8ed184341c167c6cfcd257f163e5292d5dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005448, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.3175878380425274, "model_name": "gpt-5.6-luna", "output_tokens": 56, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:35.113566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "94885de2-2291-4269-b3d0-b1913489637c", "result_sha256": "a758d6c3c4ed252d56f039a05d79cb1a711b764e8cd832f7ef6a9e2a0f1d02be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005412, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 3.629758076975122, "model_name": "gpt-5.6-luna", "output_tokens": 53, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:09.150052+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "21525222-f641-4f57-921e-6cea7142cfa3", "result_sha256": "3f0e08767838a9e457f07dd45b74a1ce066c1d09e8186e882e56e7d5509b9d7b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.43936117796693, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:35.142824+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b64b817-f895-40cf-9a65-fa70a04f4126", "result_sha256": "249f24ffc7f21ba87236bb949610f047faf36e89a2726f89bd44f1a781580d0b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.6026723940158263, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:19.771183+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5925795a-f739-4591-aea2-87a9c3bcfe8c", "result_sha256": "a6a797d9a9383510d5bdfc2ff316655d18a710bc85659720ade6d1c2543ad267", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004944, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 0.7782481519971043, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:56.656374+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "bd027342-84b2-48b7-bd63-33bb8373e9ed", "result_sha256": "551b2d2f1f8edcc9a1df5a4b263fb4acaa5c8cef24d711f0fa37d7239fa8bf33", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0483158500865102, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:08.777378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c5dcf0d0-57ee-4bd8-b303-6588f05d2650", "result_sha256": "dd8d245431703386d07721ff3f74544bd19c5cf2cc91caa685aa23f873fb7e74", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.9520876680035144, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:26.492195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "55d806f1-a198-4d57-a7af-d564d48c4a26", "result_sha256": "434327f3e5b395b7fe01bedcd09b667a0de0159e5203a1dd2303cabb5b250ae3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9511874349555001, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:09.407193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a87bb156-43ad-4caf-b462-026b9a377833", "result_sha256": "037144a7413bb694592ffc9f025277f530ec779b41f377d332cf7ff9cad320bb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0596769890980795, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:41.817647+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a59352c4-9d40-4761-ac7b-9e9cebe6886c", "result_sha256": "e64ca9bb62a022ee41bfa2eaa0c0ffd71357582e70a39df40c2bb480f9535031", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0086786360479891, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:46.821549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "101e8e1a-ec86-4909-b006-a1879064e18e", "result_sha256": "1c24c36817f15f7eebb649ac6a2f12264672373b5af31981d270eb951a04b120", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.8520850980421528, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:14.177016+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d57e7f97-f6a1-4231-94a7-5b1abdf6aaf6", "result_sha256": "cd2172ac00bd5e2c2f770a0a9068b80abeaaac91381f3f9b4a82a343369ae719", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0044728110078722, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:55.028738+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f65cd56-ecad-4888-8450-1664dda0086f", "result_sha256": "ae7166baceeb69d4031dd78e93b79a4c720f96dfda3a3131ec830cccab38af85", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0157816070131958, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:59.108495+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5ebcf80e-d466-41d6-8202-df64d7d93737", "result_sha256": "2683f34fc07cf6ffc158e7b842efe26485882de0d1ea45e271f9ec91441da4df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.7311101240338758, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:39.412626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "62797e26-2bb4-446d-9108-a3702337f300", "result_sha256": "a3cf5a6b9cd1109d665337564977179f4e5c5ceb15d79e638a64aef0a2ee9d66", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9920257370686159, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:28.961369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "b66d1ee9-dd0c-4d5c-82d2-7ba1f14536d3", "result_sha256": "d74ec98c5422d02006316376907ea713065d3036a761db3e66b1f3e9fc927538", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9596412770915776, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:33.237578+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "ffb27878-0092-429c-8e96-bbad31543630", "result_sha256": "bf098aef92249b38748435f0b58afee7b9f8b2f67b355c20a08395cee22f9f4d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.7836950049968436, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:20.535045+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8b6f834-0110-4f27-8fb7-6f1dc6724bf2", "result_sha256": "61031f00240c709bebac212aa55559501708f564ee9b06d764fb8fa820b4ec3e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004936000000000001, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.0226207140367478, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:02.964614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "ef6ca71c-a6db-441f-bb5a-c64a02fa43a5", "result_sha256": "4b082fa82ea22677001ac90c0d18c0861c48db22fbc0d6b9e2769064e481cc98", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005824, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 5.006178608047776, "model_name": "gpt-5.6-luna", "output_tokens": 88, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:33.841657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc63a95e-dfa3-4e0f-8fbc-6bec203543c2", "result_sha256": "df37b3f0b469663381eb94c6b0c88fa547e2cf3f520f4b1c1bb075ccc2d042e9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005692000000000001, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.875215887092054, "model_name": "gpt-5.6-luna", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:14.345883+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8ca48553-681d-45e0-98a6-0cf4dce26347", "result_sha256": "2a564952218c543ed0754636feb7cc3a1429fa41ac6c6249322a9292b5ed89b4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005668, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.375773865962401, "model_name": "gpt-5.6-luna", "output_tokens": 75, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:30.944526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5592a969-511f-4dbd-b8d5-c13f03dcd7cc", "result_sha256": "e757bf7817c6a455d775a98ca49fefc99d0277cdb292d6816250d7b06d835cac", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004936000000000001, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.166809018002823, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:28.293162+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "b061679a-07d0-470b-8c96-8c4fb06ff54a", "result_sha256": "a3f85b45f52f860c3586ec1b2ac2f6e0ec1d0f544897a956293ea0e04520abd4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005776, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.163878130959347, "model_name": "gpt-5.6-luna", "output_tokens": 84, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:41.300357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "dff74d9f-c71a-4a83-8c22-b20c268a3bac", "result_sha256": "f41b1a3f235c0c1e32a2b20802111653e9121c128b3d5d18917dd10176a709c7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000562, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7746451859129593, "model_name": "gpt-5.6-luna", "output_tokens": 71, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:58.814170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "04da0e49-83de-48b8-b2c3-7770e9fb4a0a", "result_sha256": "c7172918e6bbe55dc18e72dbf67826f21e0f49428eac4d3fa4e60a99101b782e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005727999999999999, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.3913976489566267, "model_name": "gpt-5.6-luna", "output_tokens": 80, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:26.994514+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "bb7f962a-8d8f-4457-8a7b-52a82cdc9be5", "result_sha256": "3dcbae87d02811d96ee94474c10eb658abf3ec8cd04e7e6977efe7aa4b41d299", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004936000000000001, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.2781754810130224, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:38.203191+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "bdacfa9e-073b-461e-976c-3db11d8a00ef", "result_sha256": "36845d29c6a7c64fecdcd96ec58dbf602c2c021e52c7c9e02fc7f67dfd93b72f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005644, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.7708455909742042, "model_name": "gpt-5.6-luna", "output_tokens": 73, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:18.165915+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "33b47cec-074f-4673-b040-b7431df46087", "result_sha256": "b2b0f12f0710cf8ea49301edc77c5cbbd74dc418b7ca57081c813550b1163cd4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005524, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7926646081032231, "model_name": "gpt-5.6-luna", "output_tokens": 63, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:09.411441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a48c3e0c-6709-4f24-b9c1-0b42cd9454a6", "result_sha256": "97fb39a85382cc991ec2bf144a3cf90676da4c509f6659a36fb9cdda3537e0ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005716, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7227470090147108, "model_name": "gpt-5.6-luna", "output_tokens": 79, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:20.405252+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2654aeb-f94b-48a0-aea0-6bc3380d649e", "result_sha256": "054873a0bcf27178782c2fea91d0462eb6970b843535e4fac4b141d5b3ee3ec7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0245625799288973, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:04.677203+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "7e76c8c2-dce8-4402-9925-a6811440db4b", "result_sha256": "9b447522fc5813a398edf77bd8150582141ee07f94fd3e071906ff3551db3d6a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9388394460547715, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:11.503430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "41789a2a-b931-406d-8264-2babcbb62b3f", "result_sha256": "38e2b1d91b5cfa858498da8380c4809be95e4bfa86051174ea0932f1d62e67bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.7394414129666984, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:04.067853+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "00b1f43e-2fd3-43c1-aad0-0b0c95128279", "result_sha256": "458e811f036f42b8cbbf068d87b8372a8830a386a42277fc76c86b412381fc78", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2171014670748264, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:52.565706+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5d8e54ca-9c04-46ec-afa7-c1d648707e34", "result_sha256": "79ea25390c494402824fff10748051c20c443f019c0593bb4b5aba5405ded546", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.502249787095934, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:32.477438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "19fb8e03-54c5-4bd4-b27d-727ae42a6b45", "result_sha256": "d13844c9208d5c82c8421f1e1a21e020872fb59bcb4fd729fc64570219f5adbc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.769145073951222, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:44.189434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "ddc17511-a276-4d50-9620-e3521e715b4b", "result_sha256": "fea5f1e3bae69bd2f0ff8c333a41e342feb7449d45d9813242e31dcc31c573ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.8146571610122919, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:50.750257+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "9f46d937-1f16-4a82-9ff3-b96e47b62337", "result_sha256": "c42103ddeddf0e88cc4f56ede8d1774bb70a3f8f0ab06d176c1084f09784e984", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2011405390221626, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:04.146145+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "6db2263a-7279-4ee4-9cad-e2850c6afa89", "result_sha256": "0ca0bf105baf4169f2158c0bc5a7f86b29492a2382153c55c3a12030207a2e86", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.5762504839804024, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:13.227140+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "543f47d8-f083-4744-b5bd-99044211351c", "result_sha256": "862088a97e3dcac1bd19bc09fd773a92dbb39c51aacdb6b4f885c288a1aa3971", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9286804689327255, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:26.117902+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f831bd8d-b32b-41b6-a94b-5343d8f1a310", "result_sha256": "06b5ec8c7c31b4771cde0932b25286ba91b3e66df932a62bc154042273cef9ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7318052790360525, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:52.795810+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0fadc84c-c71b-4599-9527-64124f127782", "result_sha256": "3180282137bc862c789da47b7f09050bb6e34d424aec261bc5c97436a5cea662", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.8736191070638597, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:50.841375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c961aa17-81fd-45e4-9063-068a267d4ba6", "result_sha256": "2621c2b86ad6d6e0d0af4f8760c7ea863bd7afc91302400f7ad8e554512abd50", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.073725391062908, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:55.273496+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "b93ecc69-3de4-4caf-be1e-15c8f56e4d58", "result_sha256": "ad8e01038cd6826d374c3effe582dc63492182ae57b5acf050657c556306043b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 0.9079597661038861, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:09.111591+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "bf627e10-2268-401c-806a-0f12d29e3dde", "result_sha256": "7fadd86fa15fc69eacfdedba06675911f9c094392a6c0e0c4fdd22d19511ed8a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.4879448249703273, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:43.465779+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "e69698c0-9cf7-487f-9ec5-5cd12e6696e2", "result_sha256": "693e7717228657514028219b70789c3aa72028706e7f19dbf57b711c89f1c8b1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 0.8687157579697669, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:03.129047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9e62502-912a-44f9-8e45-10567feea25f", "result_sha256": "bdcd5e3a1f8079f97534d803ade352e9cd20f5bc0c88913f49710af6ff4b1cbb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.3878994050901383, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:25.232700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "213049ff-5887-4f4d-8dc3-d274c1e2a028", "result_sha256": "6117439e7c15523fbb4c52154d2084839047b8fb51bca85f22d164c8c6a8ead1", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.3650759309530258, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:18.494651+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "11c374b1-e18d-43e1-9969-0b24e823dfe0", "result_sha256": "99e6f4cef71cfc75c395562ea8ff5484cead9ec8aed0da4d2c4707d4f50ae1d7", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 3.589699932024814, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:37.420503+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d98497a3-8c11-47aa-a7cd-19436175d0ed", "result_sha256": "6463f7e136b703bae1e7982f2d06a7993de4d31482f58e1cbca467048ca26220", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 0.7854763229843229, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:39.422421+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e1be89c-7492-4986-b6fe-1d477670134e", "result_sha256": "bcfd4fc5ca9b69dfdf3babcbc702c3a9c3ff6b9dcfb1f3ab3947619bc4854fa9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 3.026083300006576, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:58.101194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "40250bf5-2ac1-4c5e-992e-966f443b027e", "result_sha256": "b67136495f1e58b6731f0bccee6497222007f59bb331524b3a0fe6dbcf5c70ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 0.9240714509505779, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:05.376771+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "fe80199b-b583-4c76-a996-0ca186e2d218", "result_sha256": "e92e1049f060da03bee389e0cabbc1fd1c7592f6e422964d7578f5fe667a7b2f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.5320883890381083, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:27.381614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "e7353464-d7e7-4ef9-b5b4-65e1a90bf2c3", "result_sha256": "20917466c60d100432ec3bb2af4a2ee81008b73d89ca034c0576a7c4856701f2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004954, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.4681544459890574, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:17.602184+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "471019aa-6af1-48fb-bed0-7aa955ad479b", "result_sha256": "95eef5b6152bb4f1d912a7c5619bea2d08a5c52a69e7d8f52a633044b6d1a578", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9086948609910905, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:42.810820+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e65ec59-19dc-4009-b4c3-6397c32907f0", "result_sha256": "422890ce19eb3c709c1ce0b35f2f0cb19e2c9e9e13e00540e9bf92fe6f09a3c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.292574310908094, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:15.893391+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d0654c1-4cdc-48ea-9082-66c63f487728", "result_sha256": "78350c13c88f98d3f439061201ea7c34165537e95c4bca14e758c6ce125d7274", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0386586269596592, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:27.756753+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "2b55b9bb-dd9e-404b-92e7-66c7f4f1b4e8", "result_sha256": "08469ac97f0b7e26376c6895c9d091453eb5659a9e275c5fc5f860c797cb9b57", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.9362348510185257, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:11.213600+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "6c42b161-ce90-4a4c-bf00-66c04b13974d", "result_sha256": "65baf520fed11fd07b4a9df3b565b47e9ff7dd7108ab03dfe757dcb508ab70ce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.8777866089949384, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:06.550317+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "64661a59-b952-4435-8c92-231cbe553a5c", "result_sha256": "16c4d47c3d9670aab25cfe429e8a66e0b110a1264c363060718569ee72286e14", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.5293723539216444, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:21.489153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "9d009d67-214f-41e0-b713-3eca09e2adc3", "result_sha256": "069b696ba2ca4bb980ae2fafc8f09755051fa3754bf3e6ce228362083ff24e43", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.384630226995796, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:43.448990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0ef1ceb7-02c7-4e9b-b0e3-cbc936778051", "result_sha256": "891f64afe456ac81ff598a6ab3d480d827bc9436ee8accd5f87ea6227ad5403b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3534253040561453, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:30.139810+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a8e2397-138a-47bf-bb58-74229e7cab1f", "result_sha256": "6cbb52198f7423fae5c5ca409231352f192867b6cc0d1d959ed23860319fd331", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 3.1527634139638394, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:48.278249+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a886f29-19ba-4fdb-b24c-c28d048d4b6d", "result_sha256": "020e794ea6a109a39dcb36ca6cbe7e4f04ea43915e08059ef3e2ad2cb5341f96", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.5394904139684513, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:22.618881+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8610375-6e7b-43f2-8a49-293a09461129", "result_sha256": "a1294830f7d55dc4302be55eda192de68049ef9d66131ac47faecd7b536aefee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.268749544979073, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:23.372751+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c36a8e20-06bb-4e83-9fad-67bceb69337f", "result_sha256": "d7aac74c8ca1d058cbc4e6cd6de56d1b2851bc1b02bd7f21cf40c3e440ea544f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.8643219399964437, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:43.860369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1bc120c-c83c-43e3-9bd0-5b5691536daa", "result_sha256": "c8807db30c998acba15a782a00ae6652b549fd05b0a68a2ca913bda7ef3a00c5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.761120575014502, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:24.852566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "967ec27c-e5f7-455e-824f-b779c201ef40", "result_sha256": "5f86408079b57b6040fccfd2787063592a2bbdec9eddae2f7292dfcac95e2cd9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005696000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 3.9597162500722334, "model_name": "gpt-5.6-luna", "output_tokens": 74, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:07.444471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "457becc7-6baa-4a06-a1ff-df44f74ddbbf", "result_sha256": "fae16736f433dad77535425703506044ea55ecf344af76eff973f9b2de489d11", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.6028654519468546, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:23.018434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "2cc4d8c6-9c09-42c4-af09-582d0808d879", "result_sha256": "ea1fb66adea12baab3b418f8ba159dad94e01db34ae2c7ef50bd7a9bf9f78aa8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.293215571087785, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:39.901076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0751c114-7d16-4850-b9be-2f426d06a35b", "result_sha256": "b15b497addcf4a8bc292e650705a1217c514e0eaf291c1ee3a3021a66e5f4abc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005672, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.3785071560414508, "model_name": "gpt-5.6-luna", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:46.186507+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "04c21740-7bf7-4361-923b-9971d29f89ca", "result_sha256": "8773c58f847029fb948ba3d29b089698f6b298206ebc8200034b91a87178f6fe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005804, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.4438952569616958, "model_name": "gpt-5.6-luna", "output_tokens": 83, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:10.171602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "23832333-2e1a-42a4-92e6-1432e7a08fdf", "result_sha256": "a53f83de84bd0840fda2fe8a71593783f9c397d625fb7a31970b5e91f370b457", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005683999999999999, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.439207509974949, "model_name": "gpt-5.6-luna", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:43.406200+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "64b93566-680e-43eb-a88a-bd3f2912285a", "result_sha256": "bedbd06214ff554ede659a32b77d953048520fe6d64718bc04027ecb877302ec", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 0.9534899670397863, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:37.589382+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d5d0078e-244b-437a-86b1-00c1c15a843b", "result_sha256": "97fb25cc4b1f63f465df8f0c43ec15b524c0ba9db44521c35b035f0d5d107f48", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 0.7966333150397986, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:59.384197+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c5c8fabd-5217-4868-b489-54cb61e62138", "result_sha256": "5f50f1c6d4ce472af5be76014c25697ca3258a040f291c6ea5207d92b47ea94b", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.2258430760120973, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:12.316121+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "60d65ad5-8842-483e-9006-d752012f2d0a", "result_sha256": "e7b7c37e704dc0bba5dbb99be4c1395694f843904e1f03f181e9fef9041f4400", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004976000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.5171551459934562, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:17.614657+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "527c4f11-fc60-4fa9-9ca9-91f2ac5fa1eb", "result_sha256": "7b4fc7bf851d33594128a80f9e85e013b9b19921019aa1963c7915f6cc6b5ab7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005696000000000001, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.5183978480054066, "model_name": "gpt-5.6-luna", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:35.785213+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8a498185-3f4e-4927-a646-b70c59d0731c", "result_sha256": "82b8e0ebd7127ecddfcb3f53622d942911eaa9fa9ef2b7334041e063e69a219b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.138207909069024, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:54.276642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b3bc344-58a5-4392-9831-e9d3aa9b2173", "result_sha256": "6154fe35b96342214c9b9e3f87c226ca8badf0cda37e70993f9313af844fbbe2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.7079283150378615, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:02.283256+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "56dcdae8-94e2-4f45-bc3c-5f4f06768874", "result_sha256": "cf62ae0e712c5b2292236fa4f87b5086b45cbaa5e4cc0bab389c04cf37eb7b54", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.9467823910526931, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:23.310072+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c04cd81f-9121-4f6c-930e-cbe0dfc998f6", "result_sha256": "ddce0f65842f57ac8dd278bee5c30279fbcda8d96407e0cf144f53c76c0d31c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1432304210029542, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:54.389562+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff37fc28-ab40-41ea-ac5f-f4be1cf43af1", "result_sha256": "0d777501bcca3d6f4164511f4465983562ea3f80f8bd8165e34a23664aec6268", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.475010753958486, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:48.392341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "ba74d9c2-8846-409a-9c5c-4edd9786a6c0", "result_sha256": "33ddb9395089a5ba44b1c914963c7167219e2edd052cf3c8a1d4ba4755ea5f43", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.241458459990099, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:21.812778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a703b567-19da-4616-b909-206092fbeb06", "result_sha256": "94f67d1dc3960edf0e20efd9aca824f6b567ffa589778a4a13205d8d3c8b5467", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.014443331048824, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:26.786654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "704ae4a8-624e-46b6-b4c9-a03610f860a3", "result_sha256": "38d491e6731a8db82901e1e91348dbaabe3b74fc8ebe0d232d22bea0e32458d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.868992252042517, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:47.653842+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "57a16522-1bca-4f41-8bee-9b8950b66928", "result_sha256": "7c1059549ce690885f2bdf7c4bfa608ac6f9a842d38e448aa4933eb003fb3f5c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.7714059749851003, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:09.296896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f4a89b1d-fc6f-4824-a5bb-be7749ed3d92", "result_sha256": "1cddfa4dd636660dbde1d7a1379b4d24ec50a025a687cae4a26ef50eebdca86f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0290358880301937, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:12.577103+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "495cb2a3-6926-4097-8f8d-8b3b20a672d3", "result_sha256": "490deae27d51479f5bcf997354a83e340b9072d9ae3921a502716fba9d2f2694", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.270972686004825, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:34.244975+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd8f33fe-ca95-45e5-bcb6-490dcbd5c005", "result_sha256": "502a12e8f53806e7168af92f09c287af8f60a2bdfd6346b61114e9a05f0bc237", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9956335800234228, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:12.055869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a239cb3e-8025-4efe-95eb-6ba6b14c3417", "result_sha256": "58b3de33044cfd923c5d04d4601da220906af02c68de8a3db8d5366d7ee1b6ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9854189930483699, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:46.167256+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "379b6c8c-fd81-4f2d-8192-5aefad58107e", "result_sha256": "a9d0988867f6e4984e8dcc2dd9a97beed81f44b45a56ed851682b34a67764c47", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.8574320260668173, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:33.528874+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "037ce7a3-e6dc-4cc7-8e2d-514dd1fca4bc", "result_sha256": "b0d7c13a5ac00c7fb38a78e4c0ead8cc98a955f8042d4bd4ca2f32bac97ab7c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.8202421810710803, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:40.540453+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "72c57e5b-4115-4943-86f1-08e9cb94dbfd", "result_sha256": "f0e13dc1ee8a5ffb75038f43339cc63b083d1150e2e84d240652f47642eecb56", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.8938366930233315, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:34.013116+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c83c7f58-8047-4d80-8cb4-a0c7de230f53", "result_sha256": "a727241140fd7230a7930f33e185747e9946c471537bf1991bc3ecaeca2d48d8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.0603672560537234, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:13.306039+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f11a401-b18e-4e4a-985a-aba8dedf7728", "result_sha256": "35716562cfe6e63457ce7a44766e70ba00b6e207e421984b9a486227f7743b56", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.296705287997611, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:16.046273+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d88411ce-85ea-4a54-a604-885d33b13315", "result_sha256": "e23b424657ec2ebf704a0595e9d39a0a3315550b0610f392b9a794594baca9fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.858407773077488, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:54.108389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "eba8798c-7eb4-4195-bcd5-3ce971f63c53", "result_sha256": "d772929368a9f30e4ca318759f6eb2404a5f9d195937b66356d2c5eedc77e9ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.846882791025564, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:46.935541+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "af64a66e-7a81-4e16-a342-742bb8e8aed8", "result_sha256": "52bab6875b963759409bf29b7807b0a03ef920b909bb83dc8567ae9c704fe28e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2992979809641838, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:29.466657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "9539ea20-15da-4b23-b075-f0414b5d5708", "result_sha256": "d6a7b1815c45df136538202861e14ebf675149e35ed4dc4d0abd3960d69bd6b0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.109672823920846, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:05.527023+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8875fd26-00c8-41f2-bd48-6b9c300cb0ee", "result_sha256": "b20d897bad349e840a6f06c8bb172d9cf82db993caa94e31648e8ef05c5ff200", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0797593770548701, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:37.992150+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "73d4a670-d43c-4461-ae1b-3b13f9901c77", "result_sha256": "2e277cfbe8eb0e4c8a70abc091ccd5e6020e3289202b51618dcdd5ca68b02ff3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.708415818051435, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:07.228295+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a5faac0e-2fad-46f7-a834-2d2e2c43bbbf", "result_sha256": "084ebaa95bb37e666cbf8d2f4a3d1eee1b9b793fccced3caf6d9dd328f267b57", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.1843049729941413, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:24.632858+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5a267580-1689-4e7e-858c-6cc791d8fffc", "result_sha256": "fa735b539adc595094bd6951653d00656696311f2d00c4ae29bed0bc62e2cc33", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005688, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 4.489857338019647, "model_name": "gpt-5.6-luna", "output_tokens": 74, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:32.622103+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a1a7e065-db7c-49a7-9141-52dc34259673", "result_sha256": "0d3f31d11cb124ed9347df5905d58a89ce663d0b8827af9759bb85cbcf55b8fa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.0443327160319313, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:28.921985+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "635d005f-4ca3-4bc4-9112-4a35ac0bad72", "result_sha256": "13dd2774a63988858b3fce7f87a1d383a8ffe2985d67c0c58dc8acf8394c2bb3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005604, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.550117710023187, "model_name": "gpt-5.6-luna", "output_tokens": 67, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:22.706554+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "936dc9ce-29e5-48f3-b436-9d83832b5ba6", "result_sha256": "97b272eba176629e79eb73ebfd5c85a54ce87c2b98cf8f3273a55dcd4867a22f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000552, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.3357815931085497, "model_name": "gpt-5.6-luna", "output_tokens": 60, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:04.194419+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "ced0a6c3-c6e3-468c-ae55-114c13f4d289", "result_sha256": "87ad7d7cd5c6cbfffd709ba21d42ec9e6f7ec71aaa145660a22c1253c5e5e2bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000576, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.1838940989691764, "model_name": "gpt-5.6-luna", "output_tokens": 80, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:04.798834+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "98637a88-d0e6-4158-aecd-a90314557e52", "result_sha256": "fee19bb1c944f2de992c0e30ae34a751c1b29b064a4b462be2fb71ecb12a72ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000564, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7913932380033657, "model_name": "gpt-5.6-luna", "output_tokens": 70, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:35.125661+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "e7a80fae-8070-4718-912a-b05983348cef", "result_sha256": "aacfbf056a6a8b6e501dfcb80ec02dc6a7e903dcf92684965f4b9cdb706703b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.794700940954499, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:23.753442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3f8da01-f9c2-4915-8f8d-ad61ba84bf17", "result_sha256": "002d41a36ceb1ca974f773c4b2ac3971d8ef7106eb56b4fdd7c1819e5a3b631b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.9770120660541579, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:03.061685+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0025e31d-ad00-4380-858b-eabf0b84d32f", "result_sha256": "5c2a88c09bbb9863bbc202e76f2df32abce420e4a3153733b15583f93b721814", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005652000000000001, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.899361048010178, "model_name": "gpt-5.6-luna", "output_tokens": 71, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:30.658131+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d6811c2c-081a-47ec-931e-458f4b031919", "result_sha256": "ce22873cc397ffba87a6a75da19d064b5b613fad4a35cfa19a6d13b23c3c50a6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.8950124019756913, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:16.882724+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff23e76f-0b45-4fb4-bc0e-d3940e6fb107", "result_sha256": "709c9259872c3fe0bdb23ba2cb4bbdc1a065cf38f7e04fe0ce81e095d97d2389", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.0590899749659002, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:43.877387+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "61e9a952-66ef-4960-84d6-9ed6a162a2d8", "result_sha256": "dc4e1fc0176f5fe17e25c3873e1ffec7bf07af2cd177412c7ad0b10f6bdff954", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005664, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.5608075649943203, "model_name": "gpt-5.6-luna", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:18.745467+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c8ea5720-d60d-4b19-9275-89f98f5d12f8", "result_sha256": "286baf550bc5f4450930135fd0db301cf06b9425cf68a2e26c167f1571b532af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005748, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 4.131165080005303, "model_name": "gpt-5.6-luna", "output_tokens": 79, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:22.117185+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "85120673-a7a7-4f22-9cb5-a65137c42030", "result_sha256": "f23ef03a5f6f184085144290d2c3e9a2d90b650d02b5120b94a13531e5424643", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00057, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.217145559960045, "model_name": "gpt-5.6-luna", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:37.210544+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d1c5384d-0f24-4e4f-84ec-73a4c2138be9", "result_sha256": "3e4208e1d99739a0587cc35e00e98f6c70ac1677ab802bab4595f285c0a4c8a1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.425033177016303, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:07.836633+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a706f17e-b0be-419b-8172-bec275dd9761", "result_sha256": "d8e611f2eecedb24272681c4b7b94e0c0956571b0c6750839af7b00c6a97db74", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005556000000000001, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.4640162619762123, "model_name": "gpt-5.6-luna", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:24.812135+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a994ae29-638b-4385-85ec-5097442b5ec0", "result_sha256": "84eb7199eb6d2cc3bc467d634b2dfd082d0dc208bdf2b28c1766688899c0a5ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.030184431001544, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:31.299628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "942f7ab7-9331-43a7-a289-697ed1109af7", "result_sha256": "13bc534124372e4764a276d2629d0e968336cd87717d139a8b98954d69f23917", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005616, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 4.105023059993982, "model_name": "gpt-5.6-luna", "output_tokens": 68, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:27.397321+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d63befa1-c35b-419f-b977-49309cdd024b", "result_sha256": "0abaabbb6d8a06bc0531a697c0c325669e6a8bcba633820d499af764d15d5a5a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.7747727159876376, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:11.368619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "2a3d21be-8cee-417e-aeb3-8965033c0115", "result_sha256": "cc5f17644b0a7a802920f06e963e647bee351482794099f21ea09b1405ecb1da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005856, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7435177769511938, "model_name": "gpt-5.6-luna", "output_tokens": 88, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:47.995402+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "e1266bdd-1cc5-4201-95c5-2514ab16bc18", "result_sha256": "f72954422137e39c565e9960d11c1f183900622e6c52f496a7e862c98dc66555", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00057, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.497011675965041, "model_name": "gpt-5.6-luna", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:00.542392+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5f6089bf-2872-4c7b-a49e-9eeb738804e5", "result_sha256": "80721fd7780330c030373152598315511010528c1da6a548fd002d1bdcf0c5b8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004968, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9611291820183396, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:28.892492+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "4489c1cc-e410-4c8f-8f42-947a5bb9fcc3", "result_sha256": "33243adc4bd97d1586591303f9c8a5d037798fd6370733d24ba457956a43a3ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000588, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.8109861790435389, "model_name": "gpt-5.6-luna", "output_tokens": 90, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:49.597828+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "59185cf1-83f1-4667-b532-92343253906b", "result_sha256": "c7218209dafaf87f58523377582c5ef561831d826e9832e788caef04db5c2f55", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005692000000000001, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.5806838279822841, "model_name": "gpt-5.6-luna", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:58.193539+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "37f5cf98-4cad-434a-a2a4-3839f24485fd", "result_sha256": "f14c33b8b8760e1937829d5848b1a2171a14f3541abc829e00d3b144ea2422b2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005692000000000001, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6436362860258669, "model_name": "gpt-5.6-luna", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:19.485192+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0fe2fd8-0b94-43d2-b56b-f50e26deebd1", "result_sha256": "0bbabe3ab684e4ea9095400676feccf591dbea2d02d88e4ce48855425e2e4a6a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005692000000000001, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.0974180650664493, "model_name": "gpt-5.6-luna", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:12.056452+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d91b40a-e0dd-41c0-8054-79778e00ab08", "result_sha256": "c230b923beb1c3fd19ad31528cc98463ebdfafe49c6550611659f761c8533149", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000568, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.9567592949606478, "model_name": "gpt-5.6-luna", "output_tokens": 71, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:47.230412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0cdd22d0-a48b-4192-a60d-8d09172d93c0", "result_sha256": "5164667dfcd9139906bcecb0cf9adf144e6db9944adbf94f570500be2cda5377", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000568, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6858870410360396, "model_name": "gpt-5.6-luna", "output_tokens": 71, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:27.310285+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "7884196c-c265-4c42-9622-768d52fe5d13", "result_sha256": "eb4a8602359739eb7c7e1d2138d1b72e7e4f28f6eef330e5c9fd37267275f4ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004996, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.174609418027103, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:32.050094+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "256e0d46-a89d-4af9-b16a-30bd7784dc25", "result_sha256": "f7ff1ff499447bd5572de58b02763b107a3caa66fea777b9690d5b76a10b9ac3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.000568, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.8027990139089525, "model_name": "gpt-5.6-luna", "output_tokens": 71, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:14.734562+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "e4d3f94a-817a-4753-bab1-6713f3adb378", "result_sha256": "6ebb300d862742f158e7b605198f5c1f044fbc41faed4f80f765fa631837d3ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005727999999999999, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 7.959214351023547, "model_name": "gpt-5.6-luna", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:50.233339+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "79fb01ae-9816-4a67-85d4-6f7ace925bb3", "result_sha256": "725cd215aa5133eb7611160f267a0f6f0796302b6af81fe076d5fe91cbe1defd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005644, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.468808818026446, "model_name": "gpt-5.6-luna", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:39.032347+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "bba0796b-b971-4f89-bcb1-2db69f223954", "result_sha256": "05eb0e3cb1e9b5a193f97dd344607bb87ccb2974604fa7cbcb75e26564e4ead9", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0004996, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.056236900971271, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:47.512958+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c9595490-beeb-49ba-80c6-9d03b70a72bf", "result_sha256": "6e2764c78d515a7f54f7f89776467d8443b3a243bd60c0b15189a26f538793c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005824, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6689251499483362, "model_name": "gpt-5.6-luna", "output_tokens": 83, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:33.241247+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5ad2b0a7-1ba5-404e-9341-8bfd676ffbf7", "result_sha256": "71afc91d04432b99d084369c0691cefa7a4964d8dc7f77831c26f949df4dbe77", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005632, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 4.537616326007992, "model_name": "gpt-5.6-luna", "output_tokens": 67, "prediction": { "choice": "C" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:04.479513+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "697d63fc-7c01-4997-9eb4-79c184622d9c", "result_sha256": "f0b03260ed68d34b5a3af0a8ba17cfefd93f0c5fc3fbd6502b42242fa3a325a3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.6683396289590746, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:25:51.374290+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "38cd76ca-5ac6-48ad-8a3c-467cb336b148", "result_sha256": "d1a6a2b9a2a3de78e6e4f18f12f05594f3f3007a1882e9b2ef66f9a64e763103", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.242056049988605, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:48.459040+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f1166305-cf0f-4dfd-8962-5f1ca1a33627", "result_sha256": "e13b013203277cb2a4d013b4ae9bb7d91ba55288a0ade4014bd565cc3afd59ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005964, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.9384014380630106, "model_name": "gpt-5.6-luna", "output_tokens": 94, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:47.565837+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1129de2d-7e4c-4ad9-9fce-00ca5f4e343c", "result_sha256": "bcacd9726575825452acfd5f8feab5d7f3230c38f42aa905d7784373f838d700", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0006228, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.332309229997918, "model_name": "gpt-5.6-luna", "output_tokens": 116, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:16.725261+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "9e7b9e6c-7a53-4e99-9bd1-e28cfe9b4d35", "result_sha256": "7d2b96d993e1f53bbeac5b338d529bfe6bcfd368ea1d910f23e4a82513055c58", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4408437630627304, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:50.660081+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "84dfbde1-c526-475f-8388-77528dc0e82a", "result_sha256": "eafa1e520a078c4e371664d0f5bb3762f8b1a686bdfc299561249c45c9ad4657", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1466336160665378, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:38.559200+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "44790442-6b3a-4786-b48b-cea858f2b901", "result_sha256": "2cec3737e4bba76dfea642da0b52bd0872f8b617439d5ace405c552037a8a754", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.002324577071704, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:14.458895+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "22c6ff32-7279-4716-b8f7-ba0a3f37ca38", "result_sha256": "e8c1f26fc8f87bd0d46e5713c530be830c491dfab12f1b00f40e21a80a5ca733", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 0.8436961839906871, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:39.515750+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "af94c773-5c15-48a6-b418-31d394c903e9", "result_sha256": "50de727afadc9b7ed2c27e11203d3efdaed2f388d1c867415da2aabb92baf09d", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1523244089912623, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:09:21.124099+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "60510e1a-c8a5-4fae-9d65-58d0856fcee4", "result_sha256": "225d201136d29bc2828dab090973bf4a39e5f020b1ff1234317bfc7bb62f4754", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0006288, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.966019746963866, "model_name": "gpt-5.6-luna", "output_tokens": 121, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:56.612555+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1023c41b-1815-4913-b26d-51db647fc8a7", "result_sha256": "dff0d1790b3114480a5fd406a1de8d4d616d36b974e402b0425ca6ca7977bce4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.3068340088939294, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:03.012067+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a63c2871-bcde-4bdc-9dee-44cba40f459f", "result_sha256": "0582a2c8d5d8a0ae6be0794d24625b7d58c5ddff31cf37b116c82ddd48cfc1fc", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.0645588070619851, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:28.534819+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "85026339-f705-4398-a56b-0845e90b5187", "result_sha256": "b9489aabe9c7338bae1ee0aaffb5a2c5e5749b68a511eb55a81cafc677376165", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.4368630199460313, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:23:06.942446+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d1b8efcd-9fe8-46d1-a3f0-bf5e212076d2", "result_sha256": "d35fee9ac287a20710bf6d3294b0fafd1fd30c5bb586ce285ec671993ff2854e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.0479819759493694, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:07.624626+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f0eb5cc5-f46f-43d9-99e7-320075e76eb5", "result_sha256": "c6a33d0b5304d7fccd0d079e84b02aa6bda6fbae6349d09e35bf463e6601a3e7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.8002635290613398, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:39.445582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0f190af4-3f95-4e58-a892-bb2dbae0e18d", "result_sha256": "4c65ad948daa70cf7a4cf1b6bcfb96479c6567e549cd3014d7d7afad0b33f51b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0005004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1708207200281322, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:58.379038+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "6941947f-db87-43da-937f-ad7a1cd0ac63", "result_sha256": "786617db14dd217124b02d60a4040526a472612f3bcd3fcfa0dc26667f432708", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.181448891060427, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:53.256383+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0cb7b5d-ccc3-4ec8-90b9-0e7d0b31c907", "result_sha256": "2bf77ae7da6854b9282ef880e99b499ea0d9d210b78c5ba5376bb276e0f03935", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.072408748907037, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:30.680866+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "812ac043-80ac-49c7-8253-498c507bbc06", "result_sha256": "5fd608166ce33002bb286455b8a48f1f19b0d7a614235bd73ca749dd0ab342ac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 0.7772563899634406, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:16.763785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "195e13de-58a0-4f84-8f59-54ab80cf42dd", "result_sha256": "42428598de70d0673aadb6c6d75757ee8c6de2c1feb9109fcc6e70623ebaf28f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 3.4891060950467363, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:29.058298+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "79e8a72b-2315-472f-9ae5-25d98728bba6", "result_sha256": "80b511ef41bad1fa0e69bcf1e6858cdc0457e48cb16714dcf6b7a50fdc2604d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.604983370983973, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:27.467188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "18437fad-bfd5-42f9-89b1-8062a7a3d7ab", "result_sha256": "864188c923961495ffd7c04a29c1c1b2ca0b08a0e9fe2e96fb08d79e33daef9f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.2328417559619993, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:33.805211+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a51267d4-8652-44db-9c24-46283da2657a", "result_sha256": "0d96e21726c5fb32e32dcbabd2467570f610d5e17ad7e6221a5c64fbf7fed5c5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1791793290758505, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:08.238378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "726499e7-38f9-4046-96b8-613013238cfe", "result_sha256": "a333b1e1352d301aa5d104887ebf139bf8cdad7961792be6247d5428f12b52e5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.093729438027367, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:47.730536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "969d3dfb-e544-4fcf-9f66-08c9553d4a08", "result_sha256": "498f9224e909125b4f22dea56b72fb13f95a6fb2715e597847887f62fffd5ed0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.0303257709601894, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:00.865977+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b9cb3a7-958e-43b2-91d1-3b4a41bed644", "result_sha256": "c545194be0a1ed1c13d33b820d5feb82a0939591fd96c93204cca9ec91b05f16", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1121679389616475, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.294103+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a6f1937-bad7-4abf-8141-31a5d20b4446", "result_sha256": "02321007d2d3e823c88d4595eab189c27d794a612654a30317e9648c82147f4a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.642505026073195, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:46.779515+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "d4e87590-9fa1-46b3-89d2-4463eb99bc69", "result_sha256": "3b47f4cdd61b82ba18ec3177e35e7448c571bf7a0fd7230faea3e0a5ded5d359", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004900000000000001, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.8356554019264877, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:59.875546+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "980b9a3c-ba88-4c45-ac59-ed1006d33d36", "result_sha256": "66802270d4b54e1d56a5670290b1683a471f98269c641e149e5ec8fc7bb72105", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.5248065560590476, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:50.244821+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "45f538ad-2c89-4cc1-b01c-ae9bf55dddc9", "result_sha256": "36008d18501f78f963a5d3ceecff91b468fe8276778853a592eeb16a98d47552", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9616555969696492, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:13.405952+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a622c89f-5d9d-481c-9a63-18000a9c47f2", "result_sha256": "f58b3c497bc53d1fbf54d811499e2d969dfb460149347c52cd360003ab8762f7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.302706054993905, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:05.430360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a27fa106-e4cc-4b1a-95ce-2eab6765d79d", "result_sha256": "c549f1f4f6b2dd4e5669a933e525a3663d69407c67ff2462a94cb7fb90bdd0ce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9125644250307232, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:08.070240+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "e712d821-a574-4c4b-a2b4-3127b688f540", "result_sha256": "71b4b8af85ca4dbc816d5a0064f7cc07f37891bbdaa5d8cbd012b0f459ffa12a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.4561704529915005, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:22.043272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f4bd96dd-c92a-44c1-93a8-b809803b26b7", "result_sha256": "7face08e1119a58995dd3767fe464d1ceb0e277c3f42ad976d5a11b3b5ea1d4a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.833894670009613, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:12.326899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "268d59cd-4d91-4375-a783-9543aebddc5a", "result_sha256": "8170911356e60366e08f959f59469b1dca4ad92c265c3db1c2adafbe852500c7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1135877210181206, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:41.046284+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f2eb5275-1853-4f0e-a3d3-d54226e12e8d", "result_sha256": "e968f8b6934343dcf7ddd5c634f12233aa78a4d293bae59849d815b6b29951ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.8426596929784864, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:51.041443+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1279212f-242b-4793-9190-50e2afe29e6c", "result_sha256": "d91258a64f22fe79564ba5ea17c9772734e9c0562e5e1dd0e58c4bea51e130ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.2717818020610139, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:25.169271+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "02b42afa-57c5-448d-b916-076599b55dc1", "result_sha256": "fa4c225ac2c241ed9afbb8dcb4ac119a732acb04b7d6b8b39d93b459bd8747dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9146756189875305, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:26.930182+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a256544-a2ee-4e89-a4ad-e557e0a11deb", "result_sha256": "ef319db6019fffefe0cfaa8db2d4853d7609fffedfa04ecc15ef5d9acaa1bbea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9257008320419118, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:52.236917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "243d8dad-b4ac-4875-879b-400e9aa8064f", "result_sha256": "89d5a1a19bf9e4e2b4de1ead0a38843350e78ef93a5e950b4fd2d083d7ed7102", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.2746495408937335, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:36.971222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "63c38b7f-8777-4b45-b3cf-7d960cd3d397", "result_sha256": "6634309d242f80633903fe8b2a3ec1e949cdc9543c9cbbf8958a4ccfc9848a84", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.8240471970057115, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:09.517894+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "79b6b09e-cdbd-4502-84f5-2b36f7ca71c7", "result_sha256": "ab67106fd6a437461c71a48cf01b0a7e9d24244bb1de736fee56156e27df55fa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9976366650080308, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:36.507130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "738554e1-9ec9-46ce-b482-7cc86ceae6a6", "result_sha256": "148b86835d58316d82d7d61c4dd9451d7b902730211659ce438a805023261456", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9742976019624621, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:07.400463+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1cdb2f54-857f-4611-9cff-c6efb69c70e5", "result_sha256": "846289decfddce021729f60ba3f733a1364ea3ffa2c760a5efbb2ecc7d8d15a9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.7466178860049695, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:17.580544+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "dcfb7a49-9ba7-4d22-b784-6603e955ec79", "result_sha256": "b49200b9223f4ec890c704e652efda31de9a6d5fd0385b6e4ac2cf172548c21d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.3804174499819055, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:17.544269+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "2b992665-c3c7-4365-8fff-fd86e098f6d9", "result_sha256": "b13018d253563ed9b1697e004bb3dfedeb7b3f63210e4a600bde217047880c2d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.6183784080203623, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:43.904522+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "53276002-7a71-4bd4-87fa-a4e386ae2d11", "result_sha256": "479b543b9fd5a9d39a1c8bb17b799e2fb3cf5086ba457ea6d72a6910b60ad4d5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9940290009835735, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:55.660552+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5fc66edf-8ead-4975-9ae6-6af7d90c596d", "result_sha256": "112c467789b619b51242c92980bfb3577ac455fd0ac671926b7224a608b88d49", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.281080739106983, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:14.230091+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "7dd73a31-45a5-4043-ab4a-b1c1c6886735", "result_sha256": "415c9fd1b5c19279594405bc3204c4fe8d89f5d55dbd246ab3caf08f78f0dfdb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9810057999566197, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:21.027265+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "3217b313-3294-48a8-bdc2-e0b4c47bbf65", "result_sha256": "c12bb618174dc695cc40ed85dac1059697bbbcc0e1ae5c09da081a2561345dc3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9905252519529313, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:37.605558+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "cc922234-ab37-43a5-a134-562b8664d16b", "result_sha256": "99a638b860191ca4f093b79b6a3d4b6ac35649b9dfb2c596565b776b817f4c46", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.189512267941609, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:26.201104+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "021f8724-9f8c-4f1f-84c1-c28d3f5360a1", "result_sha256": "8f8f5b0b8db47feab69d2115ac29dc2899322f943cbeec9514f329156a9e7780", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004942000000000001, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9255244820378721, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:24.021274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "383c89af-445e-4a34-8e01-bd00f529cdd5", "result_sha256": "16f67c48015e6df25adf31c005bb277a1af2434e3f8b3560028acdd4578b7224", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.75395717297215, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:38.139788+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "cbf85799-dc3c-4a2e-8e15-60e3be1a9869", "result_sha256": "e01f7ebef64c5cb9b377179e76eeb0ac3083b72119d8e9658ec9fe60f1fd4f1c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.143947137054056, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:42.929271+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "35e2c833-ef0c-40dd-8cea-41d4f3153b22", "result_sha256": "58f5faa3ac1720aaa4d989f1fce54a0b39eb016848c70206c2ebd470b601a925", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.2063706510234624, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:01.378464+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "a4776c21-581a-42ad-bb9b-9549e19915fe", "result_sha256": "05c39ff6c4e42a941ddcd831310dcd924bcc0f4286d0b56ac2e6a8fc7c05ca71", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.676359510049224, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:05.022232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7cf5574-cb33-4562-93e4-1850a83d168b", "result_sha256": "ce3ce0e136ef91c95236e196c8c8c9126820a35fce4a6ba59740ed8e16bdac15", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 0.7461297509726137, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:14.477942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "174a0dd6-6bfa-471a-9537-776819f53f67", "result_sha256": "fcba3969164a8b8130ce61008b2211b6f6c38ac4daca0be446b15a09ebbd5323", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.952761990018189, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:21.158653+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "f0563bf8-322b-4f7f-a7c3-de754c9db504", "result_sha256": "66188d91a610a41d17fa9090e135d4d2d7a4fc628a8ab73c08a6e4a4c1e2d344", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.405931688961573, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:03.172708+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "60864c46-14bb-4d21-a695-dc8b6ea0b252", "result_sha256": "ed7ce17a2fc1346ce67b4772aba09aeebb74050552ee7874a98a887e66ca5fe8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.8570030750706792, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:18.522639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "3020ff45-a856-4832-9a99-b9d52794b315", "result_sha256": "95b3e247b69716a1442035b36e8960566c03e4af96c0c28a2f461d164bd465f4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 0.9418522369815037, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:58.661878+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "08100313-d529-4b6d-a7f7-46103abe32ce", "result_sha256": "8e6e186d37d60cae03eeca275c5a70e6ff069f12493a70f42bcc35095b48c375", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.6037661120062694, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:34.742569+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8845db55-3801-4423-a02a-66073c552877", "result_sha256": "eeffa82a8a50006f469fe97b257bc3300e51ddf62cb791933ba00a8612ba3a9d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 0.9577505890047178, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:43.784346+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "57b9e4e1-d928-43f2-9222-cfa86e280d4f", "result_sha256": "e19a63227eba40f3745388e0613917a11ae800e3bf5f2b4eb1c010264a8be26f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.6830964089604095, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:15.531069+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf270fbc-22f1-46cf-bdbe-090d2d445abe", "result_sha256": "a2dadc9d94a47446955bbf7ef9d8f09e68295a268456418c318fe2e41340011d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.196143144974485, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.874375+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "6280b043-dc3c-40ab-9994-73ec42ae7236", "result_sha256": "b8ff9551200bb6230c30dd288c6a70100a7e1bf269dc2b6a4517d8d82a0ab4de", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.8604478050256148, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:32.703284+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f423fc4-00bd-488d-963f-7a163d433b8a", "result_sha256": "d21a23a052f46235057822841dd29f8666358b173e455d9788bfd11ba49986e5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.9463105360046029, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:58.993136+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8b2c5444-4b1a-4c9c-a4fe-a08879304d44", "result_sha256": "05b740e7ed5eb0871a52f74bb251f2dfaaabbc131df3fe7e92b690a75f5379a9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.1532358509721234, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:34.685594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "7cc01263-7708-49a5-87f5-5689797722cf", "result_sha256": "32f86ceb055d5902a6d69a68be896dd55f1ab67a7bbba7062d9bb13c0558c7af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.6565872950013727, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:44.566737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1929b252-0a72-4e7c-bd9c-8fb147958df2", "result_sha256": "54643b9e6d841a44e20681be866bb2350e5871a612a6825732636b4967e9f63f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.7873177449218929, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:40.289198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "38e845ea-e1e7-4560-b76c-a3a48eabe496", "result_sha256": "bb41f8038cd11f976063f1a3f1eae496232853059f8c53f198dfc4b2ecf22577", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.0697016679914668, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:52.130085+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "ec3801b5-03f4-452a-b908-900129c5be64", "result_sha256": "a6c0a410ec098a72db4c659466958f4656c526cbc2f1e6e3220b1cc7d3d86b34", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.0262248190119863, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:34.958221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "5853baea-cfde-43ac-a41e-55933fcf8924", "result_sha256": "0d4ac92ad3d026c862b38864e961ad86dcfabc609d6b3f8635ce1e6e249666e0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.755953288055025, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:08.081245+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b18e6ff-6766-4d02-9788-aafe0bfe20d9", "result_sha256": "eee585c447908c13542a099edda89d2530e0c61812d6c4ff5f327954a534a880", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.8889983179979026, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:55.469646+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "e1e983cd-d27b-4f8c-85d8-e01a6b657946", "result_sha256": "d7a93bfa6a1cb4d44d1f4d4bb9b1189de49a4ab42de04d833b983399dafc04ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.4185114010469988, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:42.576603+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b4ddf6d-b14d-4c78-b496-e9dfb63ef358", "result_sha256": "fae141c3e6398749b446a5389e8900cb956b4b4cd74be7d80339d5dc5f0bbd2c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 0.7704524119617417, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:51.426297+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "8bd8c240-45f5-4f4b-924b-f6b40b164200", "result_sha256": "addcc47ce2486681cd44052f79b27aa411122f6a93b76b5c039697ceef238a54", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.1817935910075903, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:06.115298+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "0befbf8f-d9dc-45ec-9fb7-76817845e812", "result_sha256": "8cd52a96fe78603e156e26abb00fffc2e55d5502b464f4baea00d39b7e072d99", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.2331470100907609, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:32.296569+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "9553f025-b7af-452b-a657-74bb47a4f935", "result_sha256": "e1ddd7227b6545c81bb7becbe84bb92c6aea96ce573ad528fdfdda7e31c2454f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0050814049318433, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:50.338897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "52981e80-0c36-4d70-8751-228202a20cbd", "result_sha256": "791ddeb55256df3e5e3b3f4d1116a6e167de450dfd20d47af7b24866c371ad9c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.1723786090733483, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:10.456624+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "ba994c62-1640-4cdd-bccd-3554aa9bbede", "result_sha256": "3f24813a170a9621bfdf1434baf56448137e4fe4d07851342ec6cc30960c0d41", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.8260056159924716, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:03.131301+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "1fd414dc-57cf-464e-8155-084d5d9dfd9b", "result_sha256": "9ee7981cbf845070e3d9b84344349643d63b0b07d7ce4bbd4771f54e64e67785", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.99956258200109, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:50.916357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "86de4bf7-41fe-41e5-8ae8-32dbf8d78455", "result_sha256": "cd4cbf6445df3d3eeac4c5dffd488828fe13dea9f89136207895e5076bc2b207", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.8649026489583775, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:18.262772+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "62beb690-30e5-49c9-8c56-6c9b346a57c8", "result_sha256": "a95fb75c5fd9e3d946f6a8e0c01089c4321397380c7e9f6625342be9ed390195", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.5539156720042229, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:06.855778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "3a5f77c4-504e-46d2-a26e-aee0b0314c7a", "result_sha256": "8f2ae3d81aa2691899726291917dbd98c4b87cd4667b4bd0dca14b5c72c24b87", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.7390963530633599, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:48.894908+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "6fbf103d-d812-4690-baa9-d4da1eb12d9c", "result_sha256": "1a03bff8250b34e039111cd0921ea86b8cbb1388c33af15ff71040eb35b03485", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.9233616959536448, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:30.921897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "dd7a3fd8-8a08-4372-bb94-9c180938431a", "result_sha256": "167dfe759e2874132218730763e28d0d4e21798453984f1e51a7607c65bfb776", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.8259955930989236, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:54.847659+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-luna", "status": "final", "included_in_comparison": true, "source_attempt_id": "88c988f5-f6da-4701-a77a-540a2e6f96ff", "result_sha256": "c739cce0263c15c4cac3ed799c057252624482d73802a158ba9001d7918f3b11", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0004902000000000001, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 3.464021837920882, "model_name": "gpt-5.6-luna", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:45.764917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "305a4904-5a07-4649-94b4-fbfd04227c05", "result_sha256": "1d3250bafa8d787b11e56c609a9125f462dce8ef75deed55a0b9302fa6123f55", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.1318980430951342, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:47.469758+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "524bd2e8-0b4f-46c9-a750-a18db3779964", "result_sha256": "3e037187367f87b884d65068fac7c2fe497225d0d8e8c4258f41836b4d71ea58", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.4698624680750072, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:14.609639+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c94c970f-8405-4bad-866b-941c4995f539", "result_sha256": "38e29e42de9209077633d7eefb1769ac543ec3fad6752a84cf7b287bcd585920", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.2277845341013744, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:08.717110+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "95ffa6cb-e497-4c29-8d1a-9b25d3312381", "result_sha256": "618ee61fb9f1494c2221efb848965d523460f057aa6996d1ae674cce261abcd5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.1984361249487847, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:11.800466+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f675fe2e-7ca1-4198-9b54-f672f246d0ef", "result_sha256": "f42c424c3eeb62e6b8a047ed95e48df777c4144d9142e2ecac3381d96743cd0d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.680896883015521, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:38.855714+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "e51f9828-eca3-4b90-a79b-f94478ea91a1", "result_sha256": "3dcbc93366d56061128a71c7d54aa00c49654926c4c2a58a9550fa07a6af0a76", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.0888699559727684, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:52.923935+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a660279f-28f8-4564-b8b9-b1ced01b7eb0", "result_sha256": "af0b882c4c97a749f607e53e7e150db88010cc21a11bc832b97276a4b0af6bbc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.1663204570068046, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:38.316394+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "be0a02c6-a798-4394-8501-2c658bbc94e3", "result_sha256": "972d89284429d97a98b41603f32cda2a3f57700698424c57907f881c1b714509", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.2461368659278378, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:57.022721+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7827caea-6f39-4dde-abc5-57712baa7057", "result_sha256": "49272dd1c147ce2f83b61642a018d65cd8b1a0da6889d68561b60288369a158b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.415822479990311, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:29.167025+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9d75e26-5a86-4103-a0ae-bcddf3ec472c", "result_sha256": "9c2f57c45b0432c0c7ce9a4fe9d714357c88ff6ee4c84552c1d4e6ea2d3dc94f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.093676015967503, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:52.598513+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1894187f-20a1-4add-94f7-29fc1bdfbe42", "result_sha256": "28fc0c3aa565ab5a55e161ce896869b3e62f65b13040edccef896b82d96e5113", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.6916830010013655, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:29.547851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7dc43366-90e3-4a01-b1b6-aaf875630e5e", "result_sha256": "b69bf444f8038495a8fc9f912dd452dc94f9f48f5abb60e9b2e2a2ea2a09a30d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009864, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.388984187040478, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:50.472736+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "2b6b019d-5fc4-4cc8-8b9f-4416cc0badab", "result_sha256": "b04e1d24ffe070b9b3b984a68a2ea96652cfd1c6cc65555e5e139881807ff9b5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.4016748709836975, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:55.357594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3086990d-3d1b-464c-9f33-78cbd28aedfb", "result_sha256": "09c88efcfa22d9885a12d0b60b2d1b763a7613a68a7920766ab7d9599d7cb437", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.114392967079766, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:59.815716+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3a16a84-84c6-43c3-a750-03bf93c1735f", "result_sha256": "b7d5dbdae6103f9fb2843bc5a1a36eb84b0127bfce6257f13cc56f03d7e3d755", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1906026049982756, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:59.912290+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "522227bd-c2a9-4203-853e-3b59cab462b8", "result_sha256": "89df0e8680fda59693c3ce3b5a50d93b274d8d9ff80096eddee9c73f50365744", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.2556929659331217, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:04.870129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "da206118-8a75-413a-a215-c239797d773f", "result_sha256": "a9ccc9e4cbccf6ea3d70e566949bc86b038a635137378ec7ec7fd1cf99ceb116", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.6931242380524054, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:10.277347+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5044b347-f806-4d49-9559-22dc54288752", "result_sha256": "df364ed4b905f403102c35e56f1b99391f49a7f94498f813117bfc5bcfe267d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 3.1095218689879403, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:11.940891+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "37fdd431-8703-4fa2-b949-001bc47d2bd6", "result_sha256": "16d492acf97fe0eeae86073d02d0702733143d1e1280801110406592371780ce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.446005166042596, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:42.960575+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e0a8fff-6ebb-4bcf-84d8-5d9d553a0ebc", "result_sha256": "d2e59d280a3a396388530bf3d1e2839ab84345190a349220458a5c6dc9118bde", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.101820612908341, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:41.385702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "e872bf2f-9c43-4e3e-8276-abd8df89d6c6", "result_sha256": "3ab3ec4cdde7508b100a2bf682aee15e44c24df00ff5b2d28c6574103d650a95", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.032517412910238, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:26.240270+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c73262a-e016-440c-900d-fceff28ab497", "result_sha256": "1f5770af46d17617bdecf361dc852a4c2df3fe813c963343e58ae180096f7c70", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.108828139025718, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:21.048936+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "570ded74-8df1-4ba8-b65b-99dc76e3eda3", "result_sha256": "c90c73cc682e63a1abcf2fc801fa054b5e10b79b70660e4f99fd43a6f6d9d49b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.9945315150544047, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:06.922984+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5bd4fdb8-a080-4068-b274-757d3998307a", "result_sha256": "7a2313afe1b08f48d4fe2b00dabdbcdc88751d3d2805b1755fc4a2515d82b80e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1047187040094286, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:32.372372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "2f5e3f42-7dd7-464b-af73-d88c9e81f8b2", "result_sha256": "999b2deae84ac68b672c668f51b4b0b891b52afd0a34653550fa8bca49196647", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 3.583853736985475, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:51.417430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e0bc3b7-ef46-4e83-ab72-793eb06703ad", "result_sha256": "91ab618fd2fa59ab26d4c307cf119610774c66e5212400557c3f8e31b4136ef0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.972695626085624, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:02.439686+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c03deef-ae35-4f14-86de-28cefc399ca5", "result_sha256": "66ebad8ad3e45979d6058b31080d274ed75193345c2fe13c05821f25f063e48b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1737439460121095, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:56.696051+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "feda06e7-376a-4838-8119-dca1c89925fd", "result_sha256": "df68bf6873da468acc63dab1fe57fe32f5fe0ca4de52ab72247efbd67a3409f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010132, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.8895959199871868, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:16.750782+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "0c0169ec-3104-4498-bf1f-98f465897541", "result_sha256": "57282229648008b7340bb136dd017e94a9aaedcc5488af65e1df9011af36d0a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.0537814479321241, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:38.343987+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "fd6128c3-90b1-4d79-9f24-277518a9b315", "result_sha256": "1a59da2701b6038bee6aeebc70a52bfc0ccd4471dc30baee2f671fcd8e8dd56f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.6740092179970816, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:10.969194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "8ad1444b-8ae6-4d17-a437-a5a71897d205", "result_sha256": "d48029e8773267d44c4c6a9c355b8eda67333db695960f113ac9d2b3127fc2a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.8377444299403578, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:29.474436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4bb2bc2d-3158-478f-a454-5e04404efeaf", "result_sha256": "fcbcb594a056b7cf4b6e2763fedc2b894d8ab4123fc2948fa2ed8bb429132b0b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.417938136961311, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:00.638242+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "603a82c2-8d7d-4ab2-a342-d93bbf7e05d2", "result_sha256": "68273800998d14b2f1adad8e510cbf42da08ef4f40ee57c20a171f844ccecb12", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.7694938209606335, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:32.743299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "56fe7d87-ced5-436b-8b65-cc36142bb6bb", "result_sha256": "17dc201772fdc3aaacf05af720ea596af8c96b4a95197a2790613cb85b8d372e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.724492063978687, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:59.569307+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c414ec0-755e-4f45-ac28-9beefd8453c3", "result_sha256": "273030f691c19d8a580be5f490a860e3752866c89dee17579192456c122954d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.6120838450733572, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:40.614920+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "008335fe-4209-4060-94cb-264c62115f93", "result_sha256": "c6f51a1ee6741d23515c6e31d73d2a5d37d3f9ea79237595c98c198208ca162c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009752, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.6729174259817228, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:16.490039+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b88e0e7-7890-4b36-8947-c09ba949a98b", "result_sha256": "5534d00684593622e4950d3998e1fdbcb4e601daed01d58e87ab3ad1dbbd02d6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011364, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.535183506901376, "model_name": "gpt-5.6-sol", "output_tokens": 75, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:44.925515+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "0f0417aa-f0b3-44df-bb07-f58a74cba764", "result_sha256": "90a1899692fa5b58332b250a23158d9abe38c62cdeca8e05bb5823a58353bae0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011424, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.4886579889571294, "model_name": "gpt-5.6-sol", "output_tokens": 78, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:32.360105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "527d9b68-51c3-4456-88ab-3e43a41cae45", "result_sha256": "75429755674608fa4f2934390c87de21114a894445c61a69aaa8f71ff34468a4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011424, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.01857228402514, "model_name": "gpt-5.6-sol", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:56.886979+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ad5d332b-3ac9-406c-a3b0-167de7dcbbf0", "result_sha256": "1790392af6fbcde039ca1b970491b35996dfe57b4e47518eae6515f5c2dc5be1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011384, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.039327295962721, "model_name": "gpt-5.6-sol", "output_tokens": 76, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:13:56.446031+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f60152b-3e20-43d5-baf9-e8fb616b9af7", "result_sha256": "2d72c05ecec06afe25ec99ae28250b499af6fafcc813373637d164e6cdb94b4d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011244, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.586192777962424, "model_name": "gpt-5.6-sol", "output_tokens": 69, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:16.427847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "dba0e142-c7f1-4615-babc-926a3cc77074", "result_sha256": "c9b5aa76767a095b702d082715c440dcf2d2b5176f29caf3579ba80f65dd1af3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010144, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.746087564039044, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:33.683072+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "225dcfaa-5b39-43cb-96a0-c2d1060a553e", "result_sha256": "1c4f0d8ce234d990dba6bbf1661eaad440f401eb979c9838d57e925699fc88b4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011344, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.017760818009265, "model_name": "gpt-5.6-sol", "output_tokens": 74, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:27.792224+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "82ba66cd-c812-4244-a33e-0c58eb1831cc", "result_sha256": "0958ca4d95a7d111d8001dc91d869609d7310d7e2fbea4348b08b882040d0ff4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010144, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.1543746460229158, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:15.256744+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f2666da0-d971-4a6f-ab55-c57920327470", "result_sha256": "6bee39027e6581ac8f3d667f7bdfea969a6f0bac84f182a582d18a810f7b99d7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011684, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.4554199189879, "model_name": "gpt-5.6-sol", "output_tokens": 91, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:23.932739+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "64428181-aa79-4877-be28-8dd8d4e64249", "result_sha256": "c7c899563f0b805dd201eba73e2954ae7e93c886150f16c4750b8d1c105e288d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011624, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.346929550054483, "model_name": "gpt-5.6-sol", "output_tokens": 88, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:23.108948+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5502152e-54aa-42cb-be98-1a36cd96c0e7", "result_sha256": "e1c8afa3f21037e2fc891a328e0abb3737245903aba5d1b145976d3f61903c1f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011564, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.0113980500027537, "model_name": "gpt-5.6-sol", "output_tokens": 85, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:49.730941+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "d956fbc4-b975-4915-91d2-0b294fa8a895", "result_sha256": "f385b20fd9c38122a83f217fd25fade2c15c388524d5ace239a410188d38f6e7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012344, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 4.153251739102416, "model_name": "gpt-5.6-sol", "output_tokens": 124, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:25:55.776242+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3694ca9-9f44-4891-a77c-6ecc33640f88", "result_sha256": "e72814c8b9fd6ac603bfbd77d4e44ac163f891785a5db38bf5fbeec863aa3097", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011924, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.8844632350374013, "model_name": "gpt-5.6-sol", "output_tokens": 103, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:26:51.228820+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "8f62259a-2b36-41ce-a735-757eaf401df1", "result_sha256": "c65c40241a99f810903d91a9c4e063755914408560095aa1fb44c2ab8cce7b1b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011564, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.23754173098132, "model_name": "gpt-5.6-sol", "output_tokens": 85, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:45.930222+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "cfb9b591-2e6e-45c8-80b1-3b217341f468", "result_sha256": "cbcabc387e1f4edd94dc9f3589faee24708253caa520bce22f5f6fd208372159", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010144, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.6039394800318405, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:34.722112+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea386ae5-8b67-4a52-9666-fdd016de42d6", "result_sha256": "5dad8f2718c131eb44b929d37e19ff52a76cd9f21353ddf8e8136d7044fe02f1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011824, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.1763523479457945, "model_name": "gpt-5.6-sol", "output_tokens": 98, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:22.875658+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "6c29c2bd-5389-4ef2-bab3-78b870368d38", "result_sha256": "23e3e7840c82b58623d5056c3f83502da68ab047c3546730760bcaf0ac34434d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011404, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.3151222029700875, "model_name": "gpt-5.6-sol", "output_tokens": 77, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:39.957938+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "098c8e7e-4e87-4586-bbe6-c9d4e21670d0", "result_sha256": "e16bd2585912087df6532e66716baff27e09d9ff30182120074c039397380ad0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011344, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.0623177570523694, "model_name": "gpt-5.6-sol", "output_tokens": 74, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:24.379959+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "9439322a-9611-4345-9734-a57bcf97d196", "result_sha256": "285bc6f997874f8d9025c56089510622b069655f4c5d196a9447fcea6d5123f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011264, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.067560314084403, "model_name": "gpt-5.6-sol", "output_tokens": 70, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:02.198755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "11d2b09d-d811-474f-9ede-e4a3feeb4503", "result_sha256": "afb3858fb81d510fbb7a4a02941039a0a995223b2a54a58e75416391905f122f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011284, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.9874565299833193, "model_name": "gpt-5.6-sol", "output_tokens": 71, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:02.913080+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c2248e8-b48f-4591-9686-0903fcffdce1", "result_sha256": "fb5a0d0eec2e576f8252dc7168d4799b5f2422f979644b15b7397a4491e5a164", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011424, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.1914610990788788, "model_name": "gpt-5.6-sol", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:55.721700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "caa04fbd-e86f-424b-be64-5210777ced0c", "result_sha256": "6eb9d83f605e3244d210cf4954ec07be48ab34d7f63e99afde5696a5e68e2ffd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011284, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.7015887949382886, "model_name": "gpt-5.6-sol", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:07.731886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "67e7b8a2-b2ef-46d2-91b2-7dcdf081c72f", "result_sha256": "e1618cc7555e9bdfcce9925818342c8369e046ae186bdfe0359749814884acbc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011284, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.093808191944845, "model_name": "gpt-5.6-sol", "output_tokens": 71, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:33.627217+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "8812aa9b-4522-434f-952e-87c3087dfaf2", "result_sha256": "8b57d3aae48d7dff289db165c82bf2e1b3096bf0043714c58dd3b7b2049170e7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011264, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.6082303690491244, "model_name": "gpt-5.6-sol", "output_tokens": 70, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:55.442273+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "086a5830-a161-4fa2-b567-4829b85ef014", "result_sha256": "4be7ffde613fc5784b1a0513b7de536bd7f66002f098869c1c5c46f971daaaa8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010772, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.6022517569363117, "model_name": "gpt-5.6-sol", "output_tokens": 61, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:08.941933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "023cadcc-52cb-4700-9590-0df3f3950414", "result_sha256": "a5ff40370b62cdf01ee92e76c7df114e917304393039271a14a586210efd0545", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010892, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.050392927019857, "model_name": "gpt-5.6-sol", "output_tokens": 67, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:52.766973+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "497fb8ee-b4bf-45ab-ad92-bc192d8a09ff", "result_sha256": "ff617385addc4a718d703246bc14b6b13488ba736ce204859c3f880657511c0f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010712, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.210258510080166, "model_name": "gpt-5.6-sol", "output_tokens": 58, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:27.113589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c96ac028-d6fd-447b-89da-0d896896c1a5", "result_sha256": "2242f29d5eed6876afe1ebae985b9a7f161f2eaafce2dacefec0909f111a671c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010632, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.3149995249696076, "model_name": "gpt-5.6-sol", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:53.069822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a1715b63-f9a9-402b-b49c-fb609541dbd0", "result_sha256": "cdbc56addd895326c8c01efc0e3c1904ab162b6e653e4120d8692a427dac4d5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010732, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.9212906419998035, "model_name": "gpt-5.6-sol", "output_tokens": 59, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:00.701672+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a8fa937f-ab34-4b68-a7a5-5c1e6f0842c2", "result_sha256": "a872aaec65abc0133e3c50d1db67e9772d96855668e360d751e85f1448dc541e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010732, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.825886646984145, "model_name": "gpt-5.6-sol", "output_tokens": 59, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:00.067463+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ee4a3582-57dd-4e2f-b0d8-2dfd82ed5dde", "result_sha256": "5d0ccdd136e86aeeb57c85b9916d2b87c9d38ae1bb7032a6b6e7625e54becea5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010692, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.8309030949603766, "model_name": "gpt-5.6-sol", "output_tokens": 57, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:18.580400+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "6786cbe8-f543-4551-a681-9fb855a08a28", "result_sha256": "5422b57a7c2bf2d3598db4f8b0729498b278cebf2eef65f5338a9f1a9cee7fc1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010932, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.935380549984984, "model_name": "gpt-5.6-sol", "output_tokens": 69, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:35.593411+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9ff5deb-7560-447f-acf7-3e3656748de2", "result_sha256": "39e0ebe5e7709586a242a2327f68366f6f16333009d5175e7c104e103cbd8fdb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010652, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.6688925770577043, "model_name": "gpt-5.6-sol", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:06.473360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ff39da4-d6e0-4263-a2c9-53a5ba47d545", "result_sha256": "1695f3033885150fbd35e57d08763e124774661f509c1c6bc44efb358d84420c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010692, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.232412977027707, "model_name": "gpt-5.6-sol", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:33.456101+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "385ba468-2c10-43e6-8e60-8e5a7fc42178", "result_sha256": "1e6296a025a094d086ea894be760df43d910254fbcd37284c521a290cc356749", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011652, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.521467344951816, "model_name": "gpt-5.6-sol", "output_tokens": 105, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:18.584713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e8642d5-0f06-44c2-b18a-9e7329f17e8c", "result_sha256": "94439bfdd69f0063dce017401079f4b58d7af5720b80d0c04a417890e09d769f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015592, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 7.569807694060728, "model_name": "gpt-5.6-sol", "output_tokens": 302, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:02.756976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "710a4658-a5cb-49e2-a487-da923b617a69", "result_sha256": "c273158cdf3ef31c0266541a9ee6e84a22e14afa1d80bcc56cb2dbdd726c360f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.106015807017684, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:08.644724+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "6acde56d-7ddf-4194-88da-3875c75173fe", "result_sha256": "8d4b88a94ee3d0d9181653e3416a91144dda7c648799236afb3579b5ce81e2f5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.237315605976619, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:23.621047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a6e755ad-d11e-45cc-a296-c564405a0c7f", "result_sha256": "73afff31d4111a46006d59497788cff5c1530f1e9c2b2d3472274040ad99dc2a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.6588401969056576, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:09.560039+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed7df855-d0b1-436a-a7db-da17f0ebb0b3", "result_sha256": "594917cc5fdedb0079b5ecaf598348b526dbb3fca34a54d64c524de44e0d2c65", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.038354926975444, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:41.622378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f11cf7a6-219a-445d-bc21-40d91f75a65c", "result_sha256": "4152e71c72037e8309522ef0a59e1bda97525165f59701f812f7070aaab269e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.3121247930685058, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:46.604542+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "0d4f942d-a3d4-4000-8d29-54b52fe263aa", "result_sha256": "7876f8da89e856053c7ee0b07e8fa16bd7f530e7e6d3647568378fb65b9e90f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0900103190215304, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:14.023889+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e6f3e5e-af26-4b76-9f94-2ce50ef6333e", "result_sha256": "86dbca1d0490ad40f69dc7749f354987254f2fb2795f5574348424a731ba917c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0807715359842405, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:54.309923+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "19d5730c-387d-48c8-a7ba-7fb9de404d8c", "result_sha256": "f04b723d389f008054ae54bb7bbf8c2bf9b6dba86f59ffe15990d6779a4e5ca1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.6493610350880772, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:59.488699+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c1a97f2-7a28-440d-876f-cfed5d804bd9", "result_sha256": "2b24275080eeade74a2043d0412963da88bf1312ce26463610761abdb18aa859", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7652569300262257, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:37.432193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "91c2fa74-075b-4e4f-a863-8ec04bce8d5a", "result_sha256": "30478708a977fd1b3443ac841c67e7cbe9ab9c61e5a78c8409c5b59659f9660a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1950134029611945, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:28.492564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "787f21f8-b673-4b83-b307-cd867a049468", "result_sha256": "3d1c7a2fa57bd7d1bc4150cc55f259d7415db23d95fa48d022d6e38a6a656528", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2981639290228486, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:32.705152+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "51dc1cd1-9e73-4548-bdb0-b6afd078bb1d", "result_sha256": "09c1d4634f657043796421e9472be8da61bda48dee17b21c4c3d135d1fce908e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0402068730909377, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:19.948682+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e42f0f0-f0ac-44c6-9e95-2e05962c60fe", "result_sha256": "be7e1e344194ff711d42dca3be0bda86592bff46db5c3546246bcf583a100f2f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010956, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.094589365995489, "model_name": "gpt-5.6-sol", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:03.928452+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "bea9dd83-cfe4-4d65-bc78-21c94f8704fb", "result_sha256": "cfa18649d8df21903d869cfc25d6ec7fad442fb0eb715159ecba91adaffbe23d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.0524746279697865, "model_name": "gpt-5.6-sol", "output_tokens": 66, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:30.060336+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "b0114770-84dd-4095-8894-3c81c62370e4", "result_sha256": "ecea8eae9308b407dcea9d01931419f122154e7d715176295d7645308159b298", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010916, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 7.509721216978505, "model_name": "gpt-5.6-sol", "output_tokens": 69, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:17.302793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "222ec057-12aa-4d18-ab08-96d95ef272e8", "result_sha256": "3c6a6e337f28b765e4cca00b873626f06ef61ae7e520a9f23f0f36882558dead", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010896, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.751320852083154, "model_name": "gpt-5.6-sol", "output_tokens": 68, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:28.643715+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1a59663-5644-4af8-acc4-fc0a9602adb2", "result_sha256": "84b322d5f761ebee0818f186a43c11b7e3d8f1c7e5a53bc70fcd0f45833b21d7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010836, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.8488486980786547, "model_name": "gpt-5.6-sol", "output_tokens": 65, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:29.486560+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa74bf7b-5013-4195-92de-21c7f28e4081", "result_sha256": "727f85b535e2441abc94a56863a72fe037543f41204e43340fae5f0eb7cc96ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010796, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.724334905971773, "model_name": "gpt-5.6-sol", "output_tokens": 63, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:38.703650+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "010fc9be-e50a-4f7b-bfc3-8c9010559271", "result_sha256": "8207447fb372f4f203ae1b304d2210e68c7d1b4bb95c7c0fe8a1a68ca6980bc5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010876, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.9372499979799613, "model_name": "gpt-5.6-sol", "output_tokens": 67, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:58.144774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f6e8d39f-c55b-4b1d-9a8a-6bccb4d25550", "result_sha256": "b54a3d1ea093711d17a344cc86bbcc6f24e440442f5776ba85cc99c9cc43c061", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010696, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.0537717069964856, "model_name": "gpt-5.6-sol", "output_tokens": 58, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:24.592015+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "24745a6b-7d5a-4663-8298-6b742ee25d48", "result_sha256": "218c980b8037249f0e187ffebbcf2e7ddcc9809697a0aa1f3fa42447f7cbff4c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010916, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.2026300800498575, "model_name": "gpt-5.6-sol", "output_tokens": 69, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:38.832613+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f19db9f1-eb1e-4ea4-9c74-2958d260e3c6", "result_sha256": "d36caf01326abe488f7271639412ac49c70854f1713ddd4d97791e6ecad2e130", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.013636, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 8.210190237034112, "model_name": "gpt-5.6-sol", "output_tokens": 205, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:21.483066+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5521fa8e-94f9-4b14-9ae9-0fecce096213", "result_sha256": "6b208c6171df0d96e6abc39c4c3ef21e6a00c8e9bf76f748b7c78514a2b2d992", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010856, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.866269055986777, "model_name": "gpt-5.6-sol", "output_tokens": 66, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:08.098757+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e0e00a5-2716-48e6-b117-f323dd9e6203", "result_sha256": "b070de239a40b3066dd1770d6f0b3dcdb602a44eaacc158624e2c43ed5c5f850", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012016, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 4.373948159976862, "model_name": "gpt-5.6-sol", "output_tokens": 124, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:21.797127+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d4f60c7-7eed-4891-9708-7b9bd4008707", "result_sha256": "8cb7ba67e0421b9e0c509867eb30303803010e5532fb3c75110c4ef0144c12b4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.6462663799757138, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:04.348324+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "73c729ec-27a2-4fca-b4e6-cf64074e1d0b", "result_sha256": "43cc6e0b4cc008ab45aa48d5baa3eca96024c1c3d7c273e988c802e5c4810460", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1043783189961687, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:10.564333+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "94cfe10e-9dc0-4655-996f-bd68be029568", "result_sha256": "cd08d1e978e75a8c2b93efbe632af6171848f79df64d4ca4a1e4f4814548ba80", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.237722622929141, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:01.562991+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "064ec666-9539-4ddb-93e1-a70de22ec95c", "result_sha256": "cb81ad65856861e67f7e11a9c0803775d01f02a56c51c9f65c9d62e4790c3b3e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.351972009986639, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:52.042820+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "8cc6ab3a-a38d-4ae7-837d-8348c6c1f14e", "result_sha256": "40cf4af94f97f56c1c7e4ae66af619cad340f84770a8c715e126727e99c4cafe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2471777059836313, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:30.900887+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa9ff456-df17-4118-b86d-9f5103a5839c", "result_sha256": "e8863e5aa1ddf817ef9fa90ffceac47385823252be855bc8ba5d1995c4339903", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1865640809992328, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:43.201358+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f24188a4-ecb1-4038-ae2d-b893b05a5139", "result_sha256": "e94387e8aad9ef5ce650819babb2d4633f47bb2f1080812a370eb84195b10b85", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.4572578649967909, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:50.533957+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "dcc72430-95eb-41bb-959c-ceec550d18c0", "result_sha256": "c44dd18aada075d7b01695106dc4df7cc2d5d28840bb40d306d8364b66a26825", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0487281819805503, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:03.574000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e2d86f7-e0a5-4134-816d-29354fe4cce9", "result_sha256": "63baaf34fa690e7c4d51d5badc293559a8d28ea45bf8bd733abf111536ad2028", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0604969849810004, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:12.173421+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c04dda89-f276-4e8f-a4e5-8f424782d2ae", "result_sha256": "0300dc958e356402f610543d0ccec73e889e0219ebd3d254593ef6dd0a8ec6d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.885169890942052, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:28.562807+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7abb9c1d-9992-4854-981b-0cd294561fdf", "result_sha256": "3870f0888ae20cbf5a9c5b23503e379cf1bc0e6f89bf52bba34d206457a874c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.19415931601543, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:52.708181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "18f451cf-f8d0-4b6c-ab3c-4134aafceedb", "result_sha256": "1550b27f6023b4eaf3aafb5041e6060ed515328f1bf197a89eb50346aaf7c0c9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.630507018067874, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:51.077910+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "32fcb481-a4c2-483c-a802-dbed32e0650e", "result_sha256": "4c03055c0d78335aa88552aa4064f3ab87bf53a96e326ea5951a7bfa24a488c9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.2993307060096413, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:53.964653+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "8b99155a-8087-4675-b698-6ea00401d980", "result_sha256": "3a3d5218348ff3f8e7e5a192b5bb63acc063e43b03c1f0f3da9f72cc564cf22f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.343289625016041, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:08.300345+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c44e77f2-34f4-473a-820b-ada40c6823af", "result_sha256": "cabb633db8a6e33bc4df18e41a721002d415cf107ed56e8b0e63c234037fefad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.4890668659936637, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:43.450282+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ae94a838-7528-4068-a5f5-1eef42cd6d41", "result_sha256": "90fecf25460f91f7bfae0dbccb007ed906956fae856427d48fd557ab39c8de88", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 4.316111703985371, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:06.079534+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "6c6d0127-468a-43e9-bdd2-52776bf0b420", "result_sha256": "5b1e889da6c1fd44d36aeb4d78a786b9e93da86764d9f8277e28c0a8ba1bc9df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.030497060972266, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:24.347566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "42588720-8e9f-4b58-bccc-76a306ef2718", "result_sha256": "fc00a72b0e2c5dce89e5b83ecca8c87dfa646cd5efdc7f6404eedd0008ea1296", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.1098455659812316, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:17.715380+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "0684f076-2820-4539-bb01-b1758586612c", "result_sha256": "a12ce0d3c8d01a4f099ee84797820ba73207963120e2a86cdfc1fb5c30bf63af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.158615182968788, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:35.373463+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b14461d-478f-40f7-b221-31d33bb04fb0", "result_sha256": "3962de139cc345c815c144a1d15f2fcd0a1fc54c7cd190f98ab894511785284f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.0064119179733098, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:39.528786+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4aa9f408-03eb-47ea-80a3-fc8174429ff4", "result_sha256": "5ed34e538cb028f0cc1e3faaeddb93821145046ff85a7c25a3e883a7ba22b92a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.6039373760577291, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:56.507849+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed223da5-8b29-4f88-a2ac-1eb0960de1d7", "result_sha256": "378d3b980d7c9b4a1d08dc2154bfad24607ba0c1f02c1128dc5be1869116fea6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.8293457370018587, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:05.804704+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "573be90f-060e-4ff3-bf25-63af752d8ee3", "result_sha256": "2c4099b08c2179d2cc7b81477bf3c7197667258a504fe0fd2db2f90297dcbc35", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.2920110240811482, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:26.904951+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "b72a43d4-fbfc-4c3d-9e40-b9ddb1681b86", "result_sha256": "ac97ed5d1b566c5991f8080e8fd2eb0a0c14809fdc93725aac6874d18de9f7e7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009852, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.1758248240221292, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:15.847553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "39426453-e846-4088-af7f-8c09319fbea4", "result_sha256": "87349ce3654aee551fa65f4aefabadb320a41f7147a8adc26d2c98298b8e6ee0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2685214600060135, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:42.647421+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3ca7f177-32e9-45fa-92ed-274140d1242c", "result_sha256": "69fc3c9c2a5b062f64f0168a0206e1f5829f241ebf21e139e7d194ff06c5ea59", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.560659519978799, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:14.592870+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3033200d-2e3d-4cb9-9fb0-09e6d9eb3e80", "result_sha256": "6444c4a5919cf295fdac07786042be591c898e92e528998a965ffb214f5d68eb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.224822020973079, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:28.652528+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3b4b63ec-1de7-4bae-bc8b-c0194d2e9a6a", "result_sha256": "be1550b8f45d3e160efd6623b911ddc5c7b9da91f7386a9a9e22dcffc23ce019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 3.0598668809980154, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:12.030020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "eb759f29-4227-40fb-82ce-19c54272cc96", "result_sha256": "41bdd95691e7d99f1f225aa667522c3740ce4c2d1f2dfc3252582107bb7ac61d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1345466529019177, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:05.831691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1fb41052-9b27-4690-b706-51fcbd7f6969", "result_sha256": "a9872a96da4d9b72157ca49ea7b99d7a05031747bb797c22ec71771767b77a91", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 6.190238424111158, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:23.874700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "abacace9-60e0-495f-b04a-562cebbbaded", "result_sha256": "2927ffb1813d00944c928c0f108c1fc4f2c63350fc23fb4ceae06f847a896073", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0608785490039736, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:41.793848+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1989666-9466-4beb-ad99-ced94fd20422", "result_sha256": "1536fbbd8ba8ce8e6a1e431b11db96b9bfab1c20dda3037101091cc1c41eef9d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3193453359417617, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:29.461485+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "e3ed8def-3196-40d0-a124-2b1f9797b5c8", "result_sha256": "8c87df887c6e34c668eb225a3289007aef875c256ca32fc9e6a7fa7928527fd2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0439608671003953, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:45.611010+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1ab20e4-5290-4028-909d-a59f953b790e", "result_sha256": "86707052f7c9257014dd7d9b00b34864d5f076ecd6f77b9ad94ad83bf1c7f5b0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.399056599009782, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:23.192150+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "48d48d43-e6bc-4cff-a2df-fe9fafd1c5b5", "result_sha256": "0b1d9c12f7700f7ceb8014f3330fc17070de65691013a62b388823290cf2b683", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1729047879343852, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:22.570344+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "81724fef-ed20-41e3-821a-5dbb1287debd", "result_sha256": "8eb9082cefa676f3ee7193fbc57d1d68c6f409c1965e886d24645607280319a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3342536769341677, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:43.241873+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7c3512b4-5968-4d9d-a3ea-a270461f3a1c", "result_sha256": "db889adaac41cba681f4438a7e67ad0b0168fbffb98348b37178dd66c7f8a5e9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010996, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 7.491445573978126, "model_name": "gpt-5.6-sol", "output_tokens": 69, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:29.245079+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c7bb0e0-ea52-442d-af53-300ed4703324", "result_sha256": "a3b139f78af20bfc51eb1b9981b7ea4c6a5231d09fc2a032fd6db880429dcdcd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009896, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.4482279469957575, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:04.543436+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "e60afca9-dc04-404b-a1c9-c04da2b5ab7b", "result_sha256": "0e3e941f563a4d4b78ee2a4620a21e24df501dc44d61e56707a2c347d0b2365a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010796, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.908884795033373, "model_name": "gpt-5.6-sol", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:22.641475+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "020ecdf2-f6a7-4663-b8a8-b8379971a376", "result_sha256": "55c2f0cf774126f4cecdabccb0879f4de4ca403fcbe3328340ff685f8512b2b0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.114373376010917, "model_name": "gpt-5.6-sol", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:39.136985+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f79b3e41-e3c6-41ea-8f9b-ddcca52caf37", "result_sha256": "4997db62271ac4f47229b4a1ab260379520847777651c0b33e7635aaacf7292f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011276, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.829498266102746, "model_name": "gpt-5.6-sol", "output_tokens": 83, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:47.453776+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "97d59c38-b9ab-450f-b642-4227e02f6bf9", "result_sha256": "4759fa4b25224fe298da8433696e91a4087c3f4ead78490f49b61d3e01d084a1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 3.3877902569947764, "model_name": "gpt-5.6-sol", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:11.235446+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "9582bbfc-fdff-4e24-886a-cf937707ad69", "result_sha256": "eaec8dc415723a991723a7e241856a3764c025aaeece1ac4ed992589910e4be1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011836, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 3.112789222970605, "model_name": "gpt-5.6-sol", "output_tokens": 111, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:44.528968+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "75f161b9-2bae-46c7-923b-a28102a84029", "result_sha256": "a5254978b0608f8f1b25b582e7b4c57a159c6c5ab8c4c6be6d98fbc9c0865fdb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011256, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.5690638669766486, "model_name": "gpt-5.6-sol", "output_tokens": 82, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:38.604268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4927f17b-f2c6-4f08-a293-de15c349a274", "result_sha256": "70a892bb49b0d3531695a67e59da8cb6abd655f78070a76cfdb26883c67dc8ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011116, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.836684979032725, "model_name": "gpt-5.6-sol", "output_tokens": 75, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:01.053452+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ce5433b-12ea-46cb-b5fa-d54544c2d727", "result_sha256": "ceb7535342d21337d64f979e7766319b855a54c48b41eca06e7a8de65fdd2700", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010896, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.105701333959587, "model_name": "gpt-5.6-sol", "output_tokens": 64, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:12.621962+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "034ae485-23c4-4590-be65-b7e5daf123bf", "result_sha256": "b3bc5790a697b8ef4966c7fa4749cf91f2aed870b2567f2b075e48aeef92b02a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011096, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.8864609780721366, "model_name": "gpt-5.6-sol", "output_tokens": 74, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:17.878590+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5eb37343-0d25-4d44-a3e3-3f83f2ecc4b4", "result_sha256": "aa96e12c96164934bcb66604d729b7e3d740cdb3314175b61af04aae8678bd2d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.37305437005125, "model_name": "gpt-5.6-sol", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:35.399617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "99bb06a7-d93a-43af-a108-871534e73671", "result_sha256": "aae8ebba415f4bc7a094360091726d394038f6d7a52574534225c00961e4cb72", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0480688950046897, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:53.545714+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "387feca2-a0bd-411c-9319-b6cd3e79c097", "result_sha256": "2987171f000323f18ca363d0ac53f5ae7f65838f4ce94def8de80e4bb7cd98ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7061888530151919, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:59.716113+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "dea8341a-e021-4e65-a7e6-a67ab0737ff2", "result_sha256": "4abfd36ad193dddf1e4790321c5ee57bd37c8cfcc150fe9801205ce104cbd25a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.120809015003033, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:21.362939+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "d6371cd0-e74b-4a74-848a-5c7cf6fd9a5f", "result_sha256": "7d6ceb1e58d7210b3991fab009999daac7fb9aac8f08d2821b781cbb5ec687fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1041931869694963, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:54.002001+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd026a89-f9bd-42e4-9a81-089f5119a017", "result_sha256": "dc0985440236e5a4ec9149d68a792dc77443c53ca8428df78bafcbabe97fb14a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0059158210642636, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:46.519409+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "df6928fc-a8c1-4989-ac77-2ebb4871bb2a", "result_sha256": "607eda54f0f74ebc15f39f436cbdaed63cc9ffd8bbdb249871574d649f33b249", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7912927459692582, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:20.660373+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c6cee646-2919-4ad2-9c29-fdd5f77ac3e9", "result_sha256": "8e70dc9519b025c93dcc4c66b5ced3e1da6c7cd0fb14af7a9dc0faf50e1880fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 4.9313152260147035, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:30.068088+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c6db8330-ea42-4b74-9f9b-e3fd789339a1", "result_sha256": "7310a1eddb0ca7a4f669a60bbfcde13904eb9682fa84cee81a58e3b152dbb088", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1389097860082984, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:45.147543+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff73a315-78e0-4edc-96ee-cf0b10f131f4", "result_sha256": "483a4e237fadec4185b3afd44898ef7a654694c736c285ff0707542df2a1628c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.300634814077057, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:09.827050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ca81b02-83cd-457c-94e9-319f1efec74c", "result_sha256": "5731217ec9b279424c2dc703187daba304a6ae41f81ecb00cccb2bff7bf1d954", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.303390273009427, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:12.421609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "0c43da40-1ab6-4fb0-b9b2-379f1060e8ee", "result_sha256": "64e08d2531ede7612530fbb5d68a522e89d95dbf5d8e1d62758b97d78d75aee6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.277066086884588, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:33.248110+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce02f5bf-cb7b-4e9b-a605-0d1d64485431", "result_sha256": "2033892d45a98791eaceb8088d366d7c40e1acadd7d9af5240d9a5e79873a2d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2041458709863946, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:11.684944+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4ce7e02-64f5-43db-b94c-ea037f7ef855", "result_sha256": "f16258324d2c9cc756d299b4d3b26bb7c6110a492004fb33c4a60bd668a2d6eb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.037947466946207, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:45.995737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "262de69a-a89c-4877-8362-17e707dce025", "result_sha256": "681d9250424c4c239404126aaa1d1966eb214e39d15de2e79693ae4d251da799", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.4117417819797993, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:33.383679+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "80f2f100-89d8-441d-b741-89f74de89f50", "result_sha256": "503fdce6bd0a0e2a2d9175a49d72659ef2374d8eafe036972b6c981efc0d06af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3471710970625281, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:37.895727+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd353427-8135-46ed-8e17-853b70952c9a", "result_sha256": "efbf763e258196ebdd96e60e6606bd7594a5ee3424748f02562de8eed53b4dfe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0093668809859082, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:32.741824+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "65ef6df9-a41b-4e03-b37e-1c7c6fa727d5", "result_sha256": "30a0e44f7439523f6535a851f32042178b4056399aa010e29a50dbb06469ea6f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.383316284045577, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:12.386741+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4ec7b666-c467-4791-8af8-b5754aeb0532", "result_sha256": "7a47990aa95638661ed112e7df9fd7d961a04c8b82bc1c3f08a88c3a5fb8eaca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.337795335915871, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:14.841905+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8a1d2d4-bd19-4193-8a28-0d513afa8622", "result_sha256": "378c6bd227515d284518a652c007e83e6911d7989e98531b0046ae4dde3de45c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.981469350052066, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:55.455695+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "937c9103-ea22-4353-84b3-f843b24fb2f8", "result_sha256": "07f0de929232947ece309100807371e9ffebd0351ae80dc0f8b08b595c872de3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.4651514800498262, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:46.325130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b041b18-7947-499d-8bc4-ff47367c6928", "result_sha256": "def37fa4ea7b7685c9a5234986c61a0cdf53c76be4b96766cb9aa5b633ec272f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9133556860033423, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:28.695172+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d3a49c6-b2a5-4f0d-9e18-fce584d8db0f", "result_sha256": "cb4829192a6ca2afe26db86fbf83412fc0d47f64f9e3c50a01e4296f048115ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3206220470601693, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:04.883980+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "0856354c-d712-41c3-bfd6-3bee24478e4e", "result_sha256": "e9ad5f35b9e8680a399c30310c900973453954254440f307611810e6dfde82c9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2694896130124107, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:37.760158+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "6cf2075e-1d9c-44bf-99fa-f56466c6db9b", "result_sha256": "f3d53d92d40031f5c6c3ceb1ed27518a141bd60e09ecb40838ca98a6f47bab56", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00984, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2054606350138783, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:05.757075+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "bfd25216-5416-4f1f-9381-39cb682eee52", "result_sha256": "50e50e1aae7c449681f87911be6d690a16ec59cc3c6fe9ad909263ad507ba2fb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.061561333015561, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:21.655618+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d16f0b4-32fe-4a49-9b9f-9430a060b361", "result_sha256": "d8a8c3513c97f9f2373be54c4eb569f6efe881c9d10e04fdb5c9b415d484c1b1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.22551342996303, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:29.328217+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "9953873c-1049-40de-99f8-ee6dbe79fd78", "result_sha256": "787e12dc618c3d3c2cce8141763e8745424232dfafca00a7a5b90b1bdc1ba2d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.4805178829701617, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:30.534743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "89a825a1-72b5-4672-a774-9953ab8161fd", "result_sha256": "1440039adeb973bb6c13d33e2b037cbc13ac171cbf49e4b490e6bfa876532936", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.4276892970083281, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:20.773640+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "0e4fb0dc-0646-4625-b6ba-46e2e636066a", "result_sha256": "c742908f96f593e99e2a6ce948e926a69addc952d5d27ccbdd434ba6eaeb76e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9405328630236909, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:03.558076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7f8032d0-642d-4238-856f-3b721740c052", "result_sha256": "20a4ff04e5a9d47e65a9002bc23576496bd49cb85ad23a0d58102ea8b27ff3ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.113031342974864, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:03.115213+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "0c867b5f-1f56-45d2-b6b9-b090c05fa4fd", "result_sha256": "155a644e9e0dea24b785f81b1f810b288a7cb470d4f706bf7adca3ec01f7753f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.1510669670533389, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:33.811372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "26c1d202-2d56-48a1-a252-3a0cfd21ac73", "result_sha256": "3538190964f2df1a0bff8d78a40a25871177b1704251de2421122238bc78fd6b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.764280627015978, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:24.934075+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "36fbf864-0cf6-4475-a48f-7013e8f6d1e3", "result_sha256": "63e844892885b0444e4849077eb338c920f71cb5a62db0554c470b99760f2abf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6176558940205723, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:02.846356+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "82e87a7a-a2a4-434c-b1d6-e80e45dd89a4", "result_sha256": "b6d9c4715560fb6ae5f7bd0ec24a832fa700fb657790ca9519024e44ae5e8625", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.2762004020623863, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:28.957864+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "805e7fd4-5c23-4aa0-be7f-98fb3c2992d1", "result_sha256": "ece627d894e507491f0f346ead3edc7d083a5f48b2d378e6d2e9749625d81c3f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.0443938421085477, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:17.339281+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "b159343f-003b-4e61-be32-c8cad99cc471", "result_sha256": "28aca2e0ed6842db4f45893092d7b88c6f5f9abb8af5807dff9cce45b452f35f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.4679506139364094, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:45.719926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "e003565f-8c34-4e9d-ab7c-39d91095382c", "result_sha256": "2b86be7d7654350b6f05f9bd31043c7f2b1ba7d3cb153bbcfc01d582cb66f47e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.1351974779972807, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:17.205965+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c041bda-13a9-4739-8c0c-d0df9ba48e2b", "result_sha256": "209d40bf9542f408277fdefbc6766645f73b7af47d2675dd0430d99b922ec7a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.0958944840822369, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:18.724624+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "028a3135-096c-4d5f-bee0-6d866dce2798", "result_sha256": "b9ac305355e40c6856e04d4d75bab619f68a9677477f36a55644a81458a2916b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.745292912935838, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:35.909426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c2b3fab-37b5-47dd-99d5-4b152534672a", "result_sha256": "b71cd37c2f7d2202a5a49ae654ca418ef9e0a1b28dc02cbb43150207b182e4c6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.9641968479845673, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:05.601588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a5c9ff02-d725-49ba-aa97-d5aec76da670", "result_sha256": "7e7ad76d1d7e4d3a511f7e5093a0af71f198b8ca705addf7d85cd4d583f25831", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.3696132869226858, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:22.291955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "61e99d63-e14d-46ad-b04f-4aab618946c2", "result_sha256": "55166d9e30211939776db55662760463a63b7ba141560eabe6dc1212ea539fdd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.113057394977659, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:31.648720+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3574d114-4d25-4641-bdc2-dd325449ad1d", "result_sha256": "cdcdf055e777f65d0e54e19204afb5aae4ad0d557ab0b9ddef94fd9182dac071", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.5060487389564514, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:23.947308+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "97a746c3-2a27-41e5-bf04-a8600d11a22a", "result_sha256": "5bbf18dfd43c29e874c03c9260f6a51255662991ae6b4dda32d0b49b61f4d80a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.137911804020405, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:12.485236+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "34799a92-0eb9-46f1-8f01-0fa923fc7a2d", "result_sha256": "53ca14815f4de6f6f948252669eb1258f026d77bb61f01d298b90c560ccdd014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.2779374970123172, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:46.884889+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed8284b9-615f-45c6-bdb7-2a1b083c674c", "result_sha256": "7afb851ac561e617e649989aa7fdfc9abb52362f5d92309b6d7d9e171b9fb72f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.268042938085273, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:57.948025+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "21a5b566-5e46-4403-a351-7d52553ab6fc", "result_sha256": "76ba201f954d5c7d8fe2215a49ba63562f8c9f4bd85bf61f1bd69b7a72821adc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.203929394017905, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:26.846839+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "581d9d3a-d3ad-475a-b686-7434f662f2c1", "result_sha256": "0516c28ec0908475c76c064d596ee37193e8cf573e1a10301e6e98588d8f531b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00988, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.113491602940485, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:48.960027+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "deede765-5f8b-41ce-a047-0e8d056526df", "result_sha256": "2fb2caa1ee38c83e6129eceb0f4f68819e3c5052410492d543b72d7a0b1915f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011736, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 3.1249799290671945, "model_name": "gpt-5.6-sol", "output_tokens": 104, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:58.575311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c4388dea-b3a6-4755-9627-6343ce67fa93", "result_sha256": "dd2780d0e4c15b7176c46f82f1f2ae181d5649819d8764c454ed3bf8c8ef1571", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.9868499799631536, "model_name": "gpt-5.6-sol", "output_tokens": 70, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:18.850777+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "fe1ce97c-64a9-4285-8b8b-0da98252d0d0", "result_sha256": "08afa65470aa92b4068470e586bcb318a3c09ea6117298deaf4b9a8d92e39d0d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010996, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6384416819782928, "model_name": "gpt-5.6-sol", "output_tokens": 67, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:11.372702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e6b3ef3-300a-4dd1-a7ca-e69a0454bd12", "result_sha256": "0980ca2abd4d18622509285be43dc87dc8d69100a11592b50bb450be94311c1f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.1177445750217885, "model_name": "gpt-5.6-sol", "output_tokens": 70, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:46.801022+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "84987354-4186-4d44-abad-bd81ab4d5f7b", "result_sha256": "0d1e132d3e49c585ec5dbd53d30cca37eb079c713c8749f780c1f458ecb35ec2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010996, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.7791223189560696, "model_name": "gpt-5.6-sol", "output_tokens": 67, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:27.258159+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7281338a-a1ec-4d0e-97b6-5fea06cea4fb", "result_sha256": "ab1b5b68f64a8bfbc593dca6b3091f8b2e061edc2f83be93c990dce530cfd0a6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011176, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 4.56689106207341, "model_name": "gpt-5.6-sol", "output_tokens": 76, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:34.380198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ae52ef00-c64b-4d7c-acc1-9a7fa10bac3a", "result_sha256": "fff7125ded4d8d87a8dbd2fc3d4f0faa68b4e868fce262f88c22c4641ccc56a4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011056, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.9232757861027494, "model_name": "gpt-5.6-sol", "output_tokens": 70, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:13.530803+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "227d4ec0-ef4f-461b-908c-006593b95c73", "result_sha256": "6e166549364c53e1ae556720ecb89eccf78df81972b1b09af9d2b75c8e1499e7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011236, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.434924514964223, "model_name": "gpt-5.6-sol", "output_tokens": 79, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:44.266858+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "da87462d-5402-4a04-92b7-a1d46fbcdf8c", "result_sha256": "fd187b14ea881e3b38bd2504874883deafd854b1d7bbc03e1be570091270af03", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.013496, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 4.505167815950699, "model_name": "gpt-5.6-sol", "output_tokens": 192, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:40.652205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "22b29d5c-ea41-481e-93f0-55e391ff3a9b", "result_sha256": "00fc534df785b137ce465e4518120c06afa51a7ad5ad7b1f5087600db612285d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011196, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.6475736260181293, "model_name": "gpt-5.6-sol", "output_tokens": 77, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:47.934209+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2737946-45eb-4a16-838e-848e9eb8ed2e", "result_sha256": "a8317f229faa74a2654249c7315190935c2e314f2cdad90c884c5edc9eb2bc04", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011796, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.9130825269967318, "model_name": "gpt-5.6-sol", "output_tokens": 107, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:33.640927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1705a497-8ab5-41a0-9bf1-edab845af9cd", "result_sha256": "ff5cfee3d7efe814ee630e3c7928beb17ea77c925a9227e79d2573efee3cb338", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010996, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.100116626941599, "model_name": "gpt-5.6-sol", "output_tokens": 67, "prediction": { "choice": "C" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:01.384787+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b322603-97fa-4bdd-adf4-57b8880d452a", "result_sha256": "e2467296f8d21255a1b6d22ab55a1fca98590b4f993a5c8b14ef3d730a2bdeab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.0906470140907913, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:25:51.031369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa620e6d-3ca2-4bd9-8a31-d8079fbf67ce", "result_sha256": "df071863e694f6737ce474a2b822d09fc0e29e0a0118b614051e6ca93f6b05b0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4192331930389628, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:47.307999+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7ed21e4-d2ad-4ffe-812a-7bf130468fae", "result_sha256": "52fa867e1e9708083e429f8f90eb21b9bd02280a9a53106d9fd9fa9892d6dc70", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.497063171933405, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:46.889122+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "20ff534c-72dd-422c-a1e2-838db08ef2de", "result_sha256": "ee577fb5d2929e32e410fa8443fd8632190e9ebbb85be622135933e2771d824f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.225820800056681, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:14.251527+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a4d1cc34-7b7c-416c-85fd-eb0adc464a1e", "result_sha256": "177f1ccd83da44929f8c0d6a28025bb1723496b9ce02f1299560f7dd65b7ecdf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010512, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.5463072420097888, "model_name": "gpt-5.6-sol", "output_tokens": 42, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:50.469381+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "92685a2e-611a-44a9-b7eb-745a0470d3c1", "result_sha256": "718e7893af37f8564fb1ddb1160ca4e208b0d4a00f0f1e013c7cb08d09aed6a4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010472, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4885022500529885, "model_name": "gpt-5.6-sol", "output_tokens": 40, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:37.981794+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed0263d4-9ed0-4e90-8048-e9b70260c135", "result_sha256": "d1b85d98657331d0e74644e30661c780dbab9e5538bb02f74f29370cdef071e4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.6568453561048955, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:15.322369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "6fbfc349-61fb-42c8-a8b2-896fbaf554a4", "result_sha256": "3be2b51a4a8ff9a48517065f266a2a58f6472d6f15d866d51514ea1b6b1c9b06", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.0242213560268283, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:39.695036+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "96a2ca55-2700-45d7-ab1e-ce5897e9b05b", "result_sha256": "e779b3c2fddd3a31b9f77f56f0f15193840f4ea66f90befa0d7c81ba9aa41333", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011872, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.881188826984726, "model_name": "gpt-5.6-sol", "output_tokens": 110, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:22.652638+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "816040fc-7b27-403c-9199-1cc0f78cee52", "result_sha256": "b95f17032e736734d9b4417ce759e8dacb1ee3ec224108111dd77fc327522cf4", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1180187891004607, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:53.056520+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "cac00f55-906b-4909-96f2-1f4347e9ed80", "result_sha256": "728976f42a103ce17756e4edc251640da4873b586f2011af5bdf316ac6cae7fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.7897953540086746, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:02.952300+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "eaf4e7ec-c780-4f11-8218-29fee784de7b", "result_sha256": "302a28028abc036478a8522fc3c4d13173432329256101deb5255bfc590466e4", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.5989189210813493, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:30.690098+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "e81c65d0-9c9f-49de-98d3-8d99c6664ac3", "result_sha256": "5f0f664315c180274b0307b2ce30507a35260765931c3a0d6ed6243d8d2849c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.3198316359193996, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:05.541640+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f81f273f-1ac7-4a14-8fee-35692eaac72f", "result_sha256": "710a012605d019882d52bce3c692503346d8ff516cf8b8b45499002ff99c2338", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1415502399904653, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:07.272830+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "77ab4188-9216-48dc-8e0d-87362b52bf5a", "result_sha256": "ff6f8189c9b458acd0f0d8f2b7352b22e8c735f0d4ddf4d4a70a78cc6f463584", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.1255538170225918, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:37.780512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "e32e1fdb-851a-4937-b643-c04685c4bbd5", "result_sha256": "595b0d1692b60069c4e57c79e666fbcd24eb347c382706ce580bc3a29927c04d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011292, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.9681470660725608, "model_name": "gpt-5.6-sol", "output_tokens": 81, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:58.470881+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7aed1663-e7e3-4790-874b-9f65780ce09a", "result_sha256": "c8abffed738a4963ca59b75e3fa0f4758cfd66ab9eb8247486be555c9a19efe4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.716109007014893, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:51.701729+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f75d345f-3534-480a-b667-883882dd6e44", "result_sha256": "772249ea4c77d052aca189757f84c3d149af0ab37cdbb3cacf88908db0949ced", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.4540060999570414, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:30.518366+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "d86b1420-0ae7-48cb-a340-adf9c871a7f5", "result_sha256": "b4eefc7d3be403b50f61db2ed26c126cf498366cc7a7b87c9c1671463f2caae8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.048786905943416, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:16.590211+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "eaddce9b-3229-4a24-8f9b-10abd1582095", "result_sha256": "570b492c4dc6809199a6c4512f7535c4c28dc6b6badb5bb5c73405b6b570fa5e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1350263169733807, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:26.460516+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "2bd6ee29-6c9b-4900-8434-c3c875bff252", "result_sha256": "b6f25649eebb55cf6df468e972eee81ffb755d815542bfc2b2f4b4589de61e5b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.741872376995161, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:26.774313+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "fa62fa70-4a11-4abe-ad12-307e5acb1146", "result_sha256": "a8c32330d1607ef2c7824b8a57b0de47d652d610981f48bef64b972d95359f0a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 3.0320129559841007, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:34.040381+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5a08b62c-5868-4128-bfe0-ae9897723fb1", "result_sha256": "792d52cc56d61f30a9cd970178b9f842f3ce7ce6144cd2ba87f1e5310be10a43", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.7064166300697252, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:07.273607+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a46faec-0e61-4ece-9aed-d6e875418e51", "result_sha256": "1d0f7f0b9af14288bb50b4089cdbca769ff955784d430a40e4eaeb45dec0cb2e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1186010940000415, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:47.572467+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4dfce1e-4199-4bec-a085-4fbe871b1652", "result_sha256": "81eeaa397dcbca7d368b5d5c11447bfb9202a337c7070f56611c106151e27cd7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.436294966028072, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:59.506808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "e5c72776-32eb-40e2-9033-84bb739afb08", "result_sha256": "111bb225cfdbfbfafcd6aa2c8758008106afdf600d1f7f2232005d57d8478c12", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.3271011530887336, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.214192+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "6c5dd3fa-333c-4429-b5bc-af0ef507b989", "result_sha256": "f6c99cf187de88e5affa1720632931130514f9585e0a103b7f346052a9048a2a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.067391016986221, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:46.920991+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "819aef42-7491-4085-afa7-01154da8f1d7", "result_sha256": "298215f46869e7edff7defef9054f0aa1aa57ac39984936c90d04b5a1f650a9b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009744, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.2083218509797007, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:57.374414+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "67ff511b-4fcc-4a9f-b1c2-bc09c99d3414", "result_sha256": "336cc6b8a641289667b59dffb2000dbe50be7bc8167388736d60fe55e4dfc5df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.331124359043315, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:49.491746+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "112c3082-b682-4666-8c9a-2a0813c3c896", "result_sha256": "b1958c558ceb7e7313d0813a38175820f4bbe9f466b7748b378c714b828a2ca6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.5523621239699423, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:13.699943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "fdf758ea-c9b3-4903-81bd-f2cdcf070f4d", "result_sha256": "830954e19d48cf72c24472a611503f179517717242a383eadd85aaf8f6033b65", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0736963750096038, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:03.294151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "dd1dbd97-b5a2-486e-a71e-0cb5489d3ca3", "result_sha256": "0bb48a0e9e2ae4c28a6cc04636b52e8a478de7dd5f381ef872e3a5a673031653", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.179823211976327, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:07.510990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7e60ab6a-1c20-4cf4-954b-747a4c333553", "result_sha256": "8615ad2137de6f610fb295b57c2d08362ae6a8bd474bcd5844a854f5b8b42761", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.488964777905494, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:20.695394+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "499752c8-2b2c-4132-b16a-e54fdc9237e8", "result_sha256": "3a9ebb1f814be508e1e7a29f5c2ce467c2fd999ac8f75e4137173eefce9f7e27", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.4093011820223182, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:12.268472+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0c49bbc-7b6e-4da3-8a68-4f83ede9711f", "result_sha256": "00a3f05c5245ae41795c0922be2f1e3636fdce6d2cf6466a69789e77fc040549", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.3734541659941897, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:41.890702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7c3d602c-2a65-413c-9cb8-037b3530b4cf", "result_sha256": "f83bcd8e2f16e478796baedf58d4d28b7a791f503bbf2701779c5b51d93aee20", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.2066786859650165, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:50.206721+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f6b4646c-d9ee-4b83-98e3-be187d49bbbe", "result_sha256": "26c9057b1e7f9b00ea611490c6223a6195cb9d4dea24d37e6c37cc49b0ed68b3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.367727342993021, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:24.595121+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "809a1042-c970-41d0-88fc-db55510d93e0", "result_sha256": "4fdc2fc612df17d6e159fb919a6fc5033b95aa3c9cda3ce1b4e516f308471e8c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7981252560857683, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:25.837471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "f0fa0204-667d-4574-bce8-567b0f62b537", "result_sha256": "41360c06790d0ef12fca47470f4714f696c8f6f7b24578a8daaba5cda8379514", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9005800540326163, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:51.544554+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a5edbbeb-0eb9-444d-ad9f-dc319f32a026", "result_sha256": "c45013581bd399cab7e64834d2254826a1a28fa4c218ecaae6acd6fa675b6817", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 6.266449699993245, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:40.970297+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7fa9b3ee-a211-4b33-9746-2b5fc1216e9a", "result_sha256": "bb62bcf0b2d62c55b491c800a4c490482fa839046d7ab6bf9f068dc1b637d976", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.5039393350016326, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:09.705856+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "c498d4c3-394f-4278-a4c2-9c25f2b20ff5", "result_sha256": "43940f00c65b58a6f9466b01113ea9339d4006f66cbf14338c675b24b7be4023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0697254149708897, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:36.345441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a794400b-68a5-49e4-afa8-2cd2baa9ced0", "result_sha256": "12e7787b90c11297712553b530168a84dd8d56ae7d02f0d4b47b774a7f905960", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.939226842019707, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:06.110597+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a9ff3987-3fb3-4ad2-b295-31cf8dc630a6", "result_sha256": "2e385b845976f63101d641a0d7944715ee0bba96741c1a7715e5a36280866183", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.3287584519712254, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:17.951941+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "9f68be40-8679-424c-9251-d93cc46a6309", "result_sha256": "0dc37a133f124c54c2c6345d34c736b3553b5a1cb4f1c4095877d1083d1f8420", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.403071120963432, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:18.106869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "07335406-791f-4f9f-a57a-3da88666eead", "result_sha256": "22b5233eb73f2077dbbbcdb3b196c9c1156ae0aad262cfb633229219976e8db0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1497244889615104, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:42.519105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "6a83bf0b-ce81-4854-aa8f-8ecccf76d410", "result_sha256": "104618cdc6b32ab7792b1decffe6b428affe1270777be914b5b469fae1335a62", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.740166440955363, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:57.234199+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1aa72f4b-9448-4af3-a764-305a08f0c93e", "result_sha256": "25b695bbb25171f7666d8e91a01e7c81e4e8eaf3a3380760c060d1755eecc891", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1210955979768187, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:13.468253+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "b49d4bce-1100-45a2-b996-6a026c6d3ca2", "result_sha256": "9efc84715902124580c5bda8ce07f419209a764933d91a69e479523217690c83", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0810740330489352, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:20.649168+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "65956d63-c24b-422e-8224-cfb699226a29", "result_sha256": "5aff47fd01fef9183022f498b11a3b1635c5f5cd84668d1f61a03f4060caa553", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.3443655730225146, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:37.039585+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1fdcd2e2-1707-4e1c-887e-4ce389790deb", "result_sha256": "512a3cd1dde033eea6e8b9bea19b220eee4f9b94071959dca59106ee00e0655d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.184689178946428, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:25.091422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "2b26621d-b8e4-4652-ae5b-c258f8089188", "result_sha256": "fc244f25a49f600b7b0893fe4d2724bf7d348646d8945f74684c9799e0a4fe76", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009828, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1055986660066992, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:23.369354+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8af7a41-a071-43b3-a6c1-a039d49edfc3", "result_sha256": "fa6bb53d7356a8b6e1d3fd4d71f73eabb9e812ecf60875831a073fd35cd9a69b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.4327031419379637, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:36.227815+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "bf4ca031-25f3-44ed-94c4-126f927d740a", "result_sha256": "d5140109713a69ecd44434a8b2be3e5a4a41e675029ca6f315963ea13d006470", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.7629120759665966, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:41.686728+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "51f5392a-543e-426c-bd1b-f27b7890e395", "result_sha256": "bfcc097a3f7b14deb0a5c0f22ac92a2a8d06fc178e3bfc5b2f59b52747ccadc1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.2841074910247698, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:00.546678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4efe81a4-c8f8-4fac-b5f0-c1b1e933bbcd", "result_sha256": "941d92a0646f67ac548f4014bc24a58766c4b2c5d449d69b0055581d88fc08d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.4929338629590347, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:03.639161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "2454bddc-6822-4c8e-a3e5-fa7ab081fde0", "result_sha256": "94512a8a625d479e9cb9e05f16cd002921feadd002ebca7aca650c50c3a395b2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.0835365910315886, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:13.734925+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "46a6a20d-1843-4837-a4e6-8d29be27ce57", "result_sha256": "d9ae63419545e68d255c660e8a18b6ae337c27e0fa91aeb9da33d3c8eaac5b59", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.6643419649917632, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:20.969127+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7664c743-7e9a-4923-ba20-b19bf3cb76cd", "result_sha256": "48ecfd82e553ffed310e19ca794c9ffa3a3d53dc7e2bb43ef3ecb6b3367287a0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.3574229090008885, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:01.723694+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "a4293963-ff0d-458b-ab08-b8cae4d3a290", "result_sha256": "f83418edcf98b0a3cfef22566f89fe28bb2a85f76f112f2e7ef162bee7b8c298", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.1277307739946991, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:17.692274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "2000d222-2bf8-46e7-9ecf-3ac2eefbc390", "result_sha256": "a44281a9b13c1ebc128a5f5fe76dfe6f9e3bdd540e909ba7de2333755f528bf6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.2036638610297814, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:58.205113+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "11dc0232-85e5-4d47-b9bf-fe4cb5af9ca1", "result_sha256": "93de82895f0f6117564fa36f069765fe13e160701af27d5f0486c932225113ff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.8537091000471264, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:34.418570+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "172b64b5-7667-43cc-bd37-203e291a0c57", "result_sha256": "28ee8d49f1aeca7dd94306c7be55838080a3b235afdce0abe9f36e0eb5c58d78", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.0251875510439277, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:42.823964+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8750109-80b1-4442-8312-904aae6375ca", "result_sha256": "0b85965984224abb7f6c05026130ed58a8bd32f03ddfb155080b82d93aa83d33", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0098, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.1018430680269375, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:14.516901+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8ced39e-b7a9-491c-b97a-7ea513672d9a", "result_sha256": "8f4c7e73a728ea97643c54298a5cef5780620024a227999f3d1c37d9c0ae02d7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.4070883790263906, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:58.266577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "089dc026-3ba9-4c64-8ed4-3b5c6768fd9c", "result_sha256": "4d388b447f9fb9c468a925151c3d10f0f202e0c1b57f3108b6f289c296812cb3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.9771269970806316, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:33.304550+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "da1ea077-c688-42b7-8316-71d20fb5266b", "result_sha256": "69ce2c82b382c86e0fbd2c06e70c7da8f9f41d1b232246b6d30acfa59ed2d07c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.4100929280975834, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:58.061709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "2744b2bb-ca8b-4f3c-86d6-80aae84a3331", "result_sha256": "5d069cb41259dda2241be9a47f3c63d38742a5ad49c51836ed596b10b96bfeb8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 6.291032914072275, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:38.628888+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "61953ecd-938f-4aaa-9789-f8015d767052", "result_sha256": "193b6d1219461653daf88c47630e018b000bb0ae5db04aced2ea7b50f9fb0505", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.2828670010203496, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:43.847151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ffcfeb96-6a6c-4ffa-a26e-204fe1fbc425", "result_sha256": "f293fdd335a830a4018bcd0943fcf8be16f8038c9b3c34c5ec329c1d205d2366", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 5.011076752096415, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:44.228308+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8c69a29-a47e-48f0-80ae-47d5e2172a38", "result_sha256": "aa5b8cd23680aa9f84a554ad8741181f21cb2023be2f025f6c4354306dce14e6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.0707319510402158, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:51.799608+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "06957a34-60f3-4765-b6c8-722b60da9662", "result_sha256": "8c22c435f384f6db0ef8d92cff116909efe00462b532accc2c7b010a3f2f0e32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.5819555569905788, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:34.763779+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "2ad17751-5929-45ac-94df-d71446e21dce", "result_sha256": "bb48c56dfaf855a6e972da979933c724335275bbfb3d08bdbd9d310c3f70433e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.6074632800882682, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:07.375743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c1f03e2-3ce9-4ddf-a93d-e420e9dce4aa", "result_sha256": "43401990f53d473e2dbf9ea174952695fde530eecc9210cdb1a3dc6e7abceab6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 3.225650139967911, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:57.134739+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed87c357-e2d9-43cb-9263-64134c00e0e5", "result_sha256": "f10aa7c890ba13805645f90aefa851abd4fe833d5946f29e4a59d9914c23bc5b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.2054755940334871, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:41.669574+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "ae3273eb-20e5-45eb-af7d-3dd74b6658e4", "result_sha256": "f4e2518511cd907bdbb86b88adbad523be2ae13a78429dc11eeaec9cf6218b49", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009836, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.171891757985577, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:51.603961+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "4cdf795b-a5fd-48e4-a530-5c93c034f958", "result_sha256": "cd0507497590a15cc31775db1ec55b35f0d45b0ea34932e764ca3146d9778651", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.9254533520434052, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:06.223738+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3e2f7c6c-9b4e-47a1-824a-c1092ed908a4", "result_sha256": "51ae1e597692b09bbd6eb4522a8658601249a76dba20a9b224004dbd86ae74b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0522649630438536, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:31.740596+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b393057-2d05-49ed-8c4e-fed72b7eb6d4", "result_sha256": "7a972067ea70fcca00234cfc6e46f8e9a35d7c2501320d51f253894a17b687e4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.2448059059679508, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:49.657343+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "3166d80e-dd82-434f-91dc-da20e97cf024", "result_sha256": "4e088be6d46585502dcfc53e6e80a3dbb42eed878d1fc837c122cc4bc8fae97d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.9277315658982843, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:10.697791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "df0ebc20-0171-4c32-8e9a-a93d656dd5c8", "result_sha256": "c04fbd3841cf2cf2cb2be4914da87b501840ffc4da13f9d97f435ffdfbc551f2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.3513225850183517, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:03.403418+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c3e6527-0c08-42d2-8bbb-13cff2683798", "result_sha256": "efc410c8913f1d2b9721ced5d0c8ea46fd8d8ecbb7ae78f191d1117c22c90dfc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0933832079172134, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:50.492360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "30292728-1062-4d44-915a-782ddbd5c973", "result_sha256": "be52a90c3fa485bae9159f5e386e13768ed554af84221f65b9bea6a6a63564e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.5393286850303411, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:18.013390+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "43b5de1b-5ea3-4373-a7e2-f25803345fed", "result_sha256": "12713299e7102e09734f1254bcaa37c05bdcc2a212022b2911984cd1d7761abb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.3925387469353154, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:06.269851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "5d9c095e-6b45-4c8b-9e5f-81dcbec0c42a", "result_sha256": "0161426b928e37b61dec83572705f2675d95a80f00c44ccddf7741c289276f77", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 3.10067811596673, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:50.754810+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7766894a-0bb7-4bcd-8326-a178d89318bf", "result_sha256": "8e3af69151afc6ecc57961a1641bef2f8e4c481fd0de47d803f9f647ea03d99d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.9892844400601462, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:30.283949+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b9a51d4-7a4e-4fe7-9e13-5d4e7c0c5402", "result_sha256": "2deace8edd6b584b23bb2f7fded1834f419aa8a01adae4e487bc13d654e29774", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 2.048129120026715, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:55.800221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-sol", "status": "final", "included_in_comparison": true, "source_attempt_id": "7fc4845e-0502-4cea-a2ce-26124be92f6c", "result_sha256": "2ad16dcf481d042061227eac69bf50ebdaaf6ae6691f1f911bd1ed1cc6e9f2c9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009748, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0985863619716838, "model_name": "gpt-5.6-sol", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:42.620433+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "bcde235d-1832-468c-87a3-c1617f9085f4", "result_sha256": "d4d6c9ebc583d135e6dd0aec3a37005a784087ebef3a2bc0e1a67d8c54c44d3c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00514, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.787017013062723, "model_name": "gpt-5.6-terra", "output_tokens": 29, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:48.621156+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "141aa563-c1ee-45cf-a320-1f96794a92d9", "result_sha256": "0ca6f4e6063eaeb7cf610b95878444585d5b1440332c52204648e177f5d476c8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00538, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.7078105109976605, "model_name": "gpt-5.6-terra", "output_tokens": 49, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:15.045995+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6bef9bae-f365-46fa-b6bc-39d428b12c36", "result_sha256": "8c359d48f95ff333bae28576f883a59f8cbfb41df5900372b2cf9d9fb2e5f51a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005272, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.9449967839755118, "model_name": "gpt-5.6-terra", "output_tokens": 40, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:09.601164+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "897d285e-3d28-4b78-a81e-56b144c1910b", "result_sha256": "ceaf51045d311930183b9f462f7aaa0097155e44ca82e71afa4a9889486895b5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005296, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 4.800491377944127, "model_name": "gpt-5.6-terra", "output_tokens": 42, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:16.097122+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5431e0c0-abc1-43de-97c9-9595f2a51a86", "result_sha256": "f5b072632905ec2f035b932fdf3a135ae4d44f4a41504859f63f1394af280a9e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00538, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 3.182056565070525, "model_name": "gpt-5.6-terra", "output_tokens": 49, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:40.167361+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6ad6d576-f1d9-4fb7-951d-37dca37eaf21", "result_sha256": "b244268d02d51ff44814d3b15a00e053e8b4f3f11b2af43b354c020e3e062203", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005368, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.3431439000414684, "model_name": "gpt-5.6-terra", "output_tokens": 48, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:53.271032+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f4b58443-a76d-41c6-9e56-9307a881281b", "result_sha256": "4a40bc7733ec70dbe2139df11494882b1df7e7a1bea0c25d8ec79c0c971e25d8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005404, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.11553031206131, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:39.998492+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d7ce494-aaf4-4c55-b431-f26c8b1cc95d", "result_sha256": "23c7b91e3786cb68b14bbe2c955ac566ab0540319c6d4238130d3aa38ddb396a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005392, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.4139444479951635, "model_name": "gpt-5.6-terra", "output_tokens": 50, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:57.376684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "358a8c7a-597a-4f21-a249-4c18299a96e8", "result_sha256": "b9afc0aea57a4c5a24abe63613415d9892f885281563d7acf4aba3cd41de9aac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005236, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.2306192690739408, "model_name": "gpt-5.6-terra", "output_tokens": 37, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:29.578069+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a54792f3-791a-4ac6-bb30-acd141b87bfb", "result_sha256": "6c36be6f42922fdce5571fa60fc7feb754333cc307fd3a829bd4b1069f0da6c9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005356, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.5401056739501655, "model_name": "gpt-5.6-terra", "output_tokens": 47, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:52.075348+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "98cc3bf9-34a6-4d36-9ba3-eb21fd64a892", "result_sha256": "343dfd0998e01b13c4d4b704f95b438ceb261f3076884de2840946ccf6b338f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005404, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.2770520530175418, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:30.054420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2575b9db-0e46-41dd-98eb-f7e2c9492db5", "result_sha256": "aa71db181a8483914938240a0f3dafd8342c4bcb13ee4f98acaf35122d8479a2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00544, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.63019905006513, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:51.157859+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9633a2f4-f395-4e29-8a79-49c58dfa9b74", "result_sha256": "e6a2018a4b733c6b4d33150b678092a3ab04e3664486ab672341826bb68b8ce6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005346, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1157187060452998, "model_name": "gpt-5.6-terra", "output_tokens": 35, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:55.274461+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d05b654d-a3e8-4b7c-a0ea-ab5a16425a6a", "result_sha256": "18189fb88e2ac906b788a7705f80f136326433a8dd4fbc22dfc6600a99d4201c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005346, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.3640940129989758, "model_name": "gpt-5.6-terra", "output_tokens": 35, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:00.902460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0315dbbc-ef69-46e8-a774-528cde78c169", "result_sha256": "68552fde6336842943d618173577d3a3d4fec5ff0f518042da9832ce48f21c10", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00549, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.5396177229704335, "model_name": "gpt-5.6-terra", "output_tokens": 47, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:00.354238+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f73847f3-d4d8-4257-a281-389874d2246d", "result_sha256": "19e947b4df7c410d11dc081276587541eb825a408b567f0a8b73f07d8329e86f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00543, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.3240930089959875, "model_name": "gpt-5.6-terra", "output_tokens": 42, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:05.036208+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f220cfa9-48f2-4715-963f-081b7a9fa696", "result_sha256": "45fc3145bf21573035ce4f832f18dcfc484c14c2c3c3a46ddd538fc6751f3ef3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005682, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.7831757320091128, "model_name": "gpt-5.6-terra", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:10.754619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "53baad7a-401d-4a81-9048-f8592d7dd982", "result_sha256": "b777e17b6a1043ea83feaa3ab8951692643cb594595eb5fb314c7f14c59f49bd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005382, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.129191461019218, "model_name": "gpt-5.6-terra", "output_tokens": 38, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:12.074638+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "024e847e-0520-4a63-acdf-df476409536b", "result_sha256": "91726af00ac4ffe3483b2378d3df1242f50a6d70b490f96dfb1ee0a3dd1ad6f3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00573, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.936040922999382, "model_name": "gpt-5.6-terra", "output_tokens": 67, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:42.834193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "506894e5-08b2-4f9f-861b-3757819f3a74", "result_sha256": "3df449e7a4a90ece79f4079eadbfbe6502155b99dd055c969b7dbdbbbe020ea9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005418, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.3410958349704742, "model_name": "gpt-5.6-terra", "output_tokens": 41, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:42.046965+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7e6c6a42-5a6b-49e5-8937-f046c14b2bb8", "result_sha256": "594ae49d705bae923921cd56c986276012b7331d31cc65dc39df9bafc3f4536c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005358, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.185327217914164, "model_name": "gpt-5.6-terra", "output_tokens": 36, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:26.038234+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b2657495-59d5-454c-9589-89a5b8d86a64", "result_sha256": "74411e921f16be4c045289abac45fb6fc0d2c65faad06ce9d943a2a0d44bff6a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005406, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.7369510040152818, "model_name": "gpt-5.6-terra", "output_tokens": 40, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:22.088263+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8351f71-599c-4919-9324-7088e8005f8d", "result_sha256": "6324eb2a27bcb72bc255b0e3b8e2fbd6987e79588717cfa921637f86d12e6aa5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005358, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.2566830880241469, "model_name": "gpt-5.6-terra", "output_tokens": 36, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:06.560709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1121a8c1-e714-404e-8f30-817a98e5ca5e", "result_sha256": "a1409dadeeae7e963e3a183e4c951ed57d2bb848eb29a68c5d23d965f1389f2a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005442, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 4.336499425000511, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:35.695764+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d1d8a6bc-5cf8-4e2a-9bf0-b05ee521c204", "result_sha256": "989ad1856f6554a7a5e87bdccccbeb15790d35f6c6e42fe81dd331e9dae70a36", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005802, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 3.7327629489591345, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:52.161335+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e1b3de7-6da6-4399-a11d-e02396af6007", "result_sha256": "c8ad5410e3a2f010fd2c064c6cb3e99ec595b5f6ba8cef1ac28dc1ac00e1befa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005406, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.367348558967933, "model_name": "gpt-5.6-terra", "output_tokens": 40, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:01.928699+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d45a49d-5a7a-4486-8115-9d41ceea0a7d", "result_sha256": "b0d0fa0492a2e2788d130996bb91e78fc50b1d3158d04f5508fb51cf9470dfdd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005442, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.1845170219894499, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:56.775600+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d1f86bc5-4785-4e6b-8b96-71ab59bfc8a0", "result_sha256": "034c736581b7462777dd789074efdb1fbb4e74dc9ed182cce558755cccbf5f43", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00561, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.758031288976781, "model_name": "gpt-5.6-terra", "output_tokens": 57, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:16.907509+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b43591da-e3b6-401c-8e28-898e27aae36f", "result_sha256": "8e13e0babb12724a23e2d7649fc7442c1b681d3d6585a733805e7140bb059ad1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.0813770699314773, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:39.231132+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d36da5d6-357f-433c-bd40-84b036f702ce", "result_sha256": "c72139e2fa963cb56ba3efe30698e56580cb47c112688eaefe7cc5437a0dd94b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.9607562039745972, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:10.508030+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c9ec14b-ca70-4f52-8fac-ece4566f7dd3", "result_sha256": "81e42a12519a82fefe656ebafe3b9dab2725c010b3e1f1abaf55d07e3b240449", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.9930016229627654, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:28.753141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "57543f8b-6736-4228-bcfe-35efdb83054b", "result_sha256": "1558f0b83f373d955b942acdff7e72586250f3592e0545f14781cb5e15216f1f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.1296736729564145, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:00.466503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "adc644fd-1562-498b-8e69-510eae7f3e44", "result_sha256": "c7d32805f07c4e6b350d1b97ad82d3378d4480152030257b27a535801862db62", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.1652702399296686, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:32.344876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b799594a-373e-4786-b765-39c2eb2d70fa", "result_sha256": "ce3cdcee250acf9732f85916e6a27b98c610615fd81af7a41b8254d162ce6184", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.1412852329667658, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:59.119084+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8f4c3286-7b64-4c81-af54-39e08b6f74f6", "result_sha256": "b54c3703c02ada62779eb0f14d5a466dc66f570a2bffdeae77e8ad4c9d06e4a5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.2372675898950547, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:40.794817+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e8c77810-df88-4f74-9c84-76d9683c1677", "result_sha256": "586a90247014c83d2e55d4b3dfa2dfb20ec072e295651c131e189c9dd24cca49", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004904, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 0.9671448599547148, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:16.086132+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4844d824-973e-421a-9008-a6d79cd8c28f", "result_sha256": "2783551b981bb63250250b788e35b47b442b54aa28835f28c67445af9ca859a5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005616, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.5025471670087427, "model_name": "gpt-5.6-terra", "output_tokens": 57, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:44.362377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c8afb8a-f46b-4cda-a84d-d870f260f886", "result_sha256": "62fb9b5619e1780fd362bea0a5f35c338c7ebfdb984f8326df0a96b4b2c25b2b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005988, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.997497473959811, "model_name": "gpt-5.6-terra", "output_tokens": 88, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:32.183696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c9e309f9-8160-4a92-b063-a9b4a982d910", "result_sha256": "51328aeabf0c73c23bfe3fea36e96398861998ed6870a61334e8e082b89ccaa4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005988, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.3062102789990604, "model_name": "gpt-5.6-terra", "output_tokens": 88, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:56.761548+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a46231f-cbd7-4a70-ad74-103de28b0501", "result_sha256": "2608720771ac271be9cbd66c671f76d4ae29976f3048eb4a043da40a97506503", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005592, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7651015940355137, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:13:55.242972+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "de2c4e0c-a160-48bc-a145-bb294143c2d5", "result_sha256": "bb0686d9530a15dd98dac43de1727ddde38a3744f1549aa2715a98e834de591b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00564, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.820615406963043, "model_name": "gpt-5.6-terra", "output_tokens": 59, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:15.859431+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a146277c-41d9-40c2-8de1-1185b7a238f6", "result_sha256": "d5b48be1004fd06ffc282c1899b5438932981647ffc10a85ff20efbd88e876dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005388, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 5.489650465082377, "model_name": "gpt-5.6-terra", "output_tokens": 38, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:35.638716+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2989d7c1-7d1f-48a3-bfe4-0933cd413ccd", "result_sha256": "0bc0551ef94642b8553dd56b6abfc2b34df6b845677475311d599b9c8522d78b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005856, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.9969723470276222, "model_name": "gpt-5.6-terra", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:25.863926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "00bfd885-f788-4da9-85ee-38028830f6fe", "result_sha256": "235db703dddfb7c14bf2065348f8c27b1841d5c8875709f185146ac63ee2b4c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005448, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.765558535931632, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:16.239677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d541e28-52bf-412a-b307-2f7cdc07cf0f", "result_sha256": "ccc64291a4fe83a082f48e1bd994a5438147c579283bb01c7fa8c4fe73d1cc2c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005748, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.2548505270387977, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:24.116370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f0d604d-edb1-496c-bd92-90f345e51a42", "result_sha256": "bb8cf70f8febf57e19c6f62fc238838238b9b0940222ec527a00b274edb5ca8c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006588, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.0746574950171635, "model_name": "gpt-5.6-terra", "output_tokens": 138, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:23.200670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a937be35-f093-430a-925b-c587392b8c9b", "result_sha256": "df6aba0fd787e979ce6ac2e90d364f3d1ba5893d9463821714d55e4b7a0453c7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005808, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.1766321070026606, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:50.060713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8dce9fee-e1f1-4c41-9615-70e96b596eb2", "result_sha256": "2bd14a65c2527a74a029968d7b09d590ec4f926d752dc67ca42e00ec0bfc3115", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005856, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.303515161969699, "model_name": "gpt-5.6-terra", "output_tokens": 77, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:25:54.074195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4507ecdf-b917-4a2e-a816-dd274c897770", "result_sha256": "fb3c82145bebb56889bd13c1d567aa3df977ee0c19004146752a1e7c0ba69a2a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00582, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.2940786159597337, "model_name": "gpt-5.6-terra", "output_tokens": 74, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:26:50.193641+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "91cbd21e-ec1c-48a0-aa52-9ee8f25fa810", "result_sha256": "ce9ffc2d8d219a2eeb25263849d8261536d9a5315a33ed2be453f2d9db94e9cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005796, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7413520780391991, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:45.174037+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "595a2b4b-0fb2-425b-8eb0-bbcf70861a4b", "result_sha256": "0c4420b1053e7e3dd1823f7f3a056895a3ea781803b367e3e24dfc4c75a173c8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006024, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.0716010679025203, "model_name": "gpt-5.6-terra", "output_tokens": 91, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:35.812471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3cd68f3e-b9bb-4e03-97cf-610f06377bd6", "result_sha256": "6b2ff6e01feb0259611a74a9aef3de8591471722b2293b57df658292a5749a62", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005988, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.1796841549221426, "model_name": "gpt-5.6-terra", "output_tokens": 88, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:23.245534+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8870b843-ea77-4902-8287-81feb3052451", "result_sha256": "d122978420d8715af8f0a4003b0708cb70df76779f1473bbcc8dbbb4d68fd5c6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005544, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.6834599430439994, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:39.651109+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d6b5ee90-0040-4539-b1d9-59eacbde52a6", "result_sha256": "00822ecfbd4b4a61836f3b8884c0b8d6eb4aabf4603f60253f92aebab4fde592", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005808, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7589094321010634, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:24.486655+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "73b763c4-8399-499a-b2dd-1b39ca4dacce", "result_sha256": "b39de2884533bc848bb5a7df68ed958b3fd632a5ef716d54bca7c000be4de214", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005832, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7159525220049545, "model_name": "gpt-5.6-terra", "output_tokens": 75, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:01.877954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a3940f3-9215-4cd0-9c36-558426452349", "result_sha256": "d39d799daf7aaa2b84b4dfd82659eebccf994584053ebda3b4b2d9c6503e384e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005832, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.9657263139961287, "model_name": "gpt-5.6-terra", "output_tokens": 75, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:03.873155+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf09cba3-09f6-4b0f-8d05-0a9c0b8cb6cf", "result_sha256": "ac668f3cc696ea2436c0f88dae688716d4bb6030d565363322efe946f6d549f3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005868, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.141844204044901, "model_name": "gpt-5.6-terra", "output_tokens": 78, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:56.629034+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d1799f3-5221-4314-93ee-f5e483d2c892", "result_sha256": "1424b226f11e2afcb9eb60d74ecbd9c77dcd7915897d767760cb573616ea23f3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006192, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.3037946169497445, "model_name": "gpt-5.6-terra", "output_tokens": 105, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:07.509274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "67525280-1f0a-4710-978b-0e73cdf3f5f5", "result_sha256": "695506c41c4a1cb3338c3721c032cc25d64d80b594cbba2950f7a244b1ddc372", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005796, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7309958960395306, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:33.403416+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0f4d8129-a3cd-4483-aa6f-ef672a983f16", "result_sha256": "2aac30ff5163102282101d6a9ea608a4b4856e3b00887f95ce94e7aa89f725fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00582, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7953735580667853, "model_name": "gpt-5.6-terra", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:53.708198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "147ee05a-d426-4b8e-886f-3cce1774c480", "result_sha256": "eebfda95f98f3ccd96b13533425403acfe845eee9f9acdbec422750cdf29bff7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005484, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.7099022230831906, "model_name": "gpt-5.6-terra", "output_tokens": 59, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:09.280143+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6dc87379-d694-4bb7-af26-170ab34a7286", "result_sha256": "61786bf3a8b3bb7f4a87dd1a123c976ceba6d649e7546f32b8d0b2f3488a6cf8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.3362305740593001, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:52.930494+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "48a9f2d3-f5bf-45dd-ba52-a46ea52d1906", "result_sha256": "39b3bfdb36da816faa4abc2f832b4ec34a74fe4bf18fd3e68361f928c2f69100", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.5382910799235106, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:26.935242+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4660032a-1cfd-4b8d-bfdc-43fe6c6d2361", "result_sha256": "2068a7a31a1a27e112fecef70fb35a0918ffd8cf7220980d03126bcd27fc5f92", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.467744673951529, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:52.302707+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "82507424-4cb0-4794-9224-cce493756a81", "result_sha256": "5c6a6cd3a4a8f01e3e6a527f5410f47010f9c86ed69238f53ae2beaca239f5c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005472, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.3850220440654084, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:59.467816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "76727905-bf4b-4f37-9013-c3697ffee86b", "result_sha256": "0337fd9175bcc17adabe621e08b9563260981a49d4fae635e4615e977bcbff11", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.4533480129903182, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:59.441677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6115a41f-bea0-4086-a54b-dc0fa9b27e8c", "result_sha256": "fb0313be917ef2944dc4a722c9a9d62514a9f4d4c4b85cf6944cee65ab54cedc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005328, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.652316435938701, "model_name": "gpt-5.6-terra", "output_tokens": 46, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:18.822344+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "003dcca1-4086-4256-bb7e-48970b0c3b91", "result_sha256": "1e0dc68a3b658a83e33ce13cb920493eb071c0577c4e36607903e2fbeee826e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005412, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.615151818958111, "model_name": "gpt-5.6-terra", "output_tokens": 53, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:35.373049+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0682761b-df4b-4dfa-a781-743573cd7397", "result_sha256": "72445e768940adac8ce7caf53d8eba242f5de8dca8ffeb7894b5396ad695de19", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.4533678960287943, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:06.388882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5135caef-ccad-470e-a593-2f34daae8d23", "result_sha256": "c8cf1650d60f5fcc7d2708837cfa714dd6ff45c80171eaca340ae0352d6e89c0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.381106938002631, "model_name": "gpt-5.6-terra", "output_tokens": 52, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:33.742693+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "07d08d30-e3ea-46e4-ac66-65f5179e9f1f", "result_sha256": "7b2a82f64fafc3768cb6fd5c2956db5509c49968818118b60d960dd8833be477", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005388, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.6457828019047156, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:17.787475+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ca5c4fa9-612e-4ac8-8f47-d57293dfabad", "result_sha256": "2328f2c5c38079b16b81cfe9880e698bd7e6f5e947812cabcf63f442ec45542f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007188, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 4.563923442969099, "model_name": "gpt-5.6-terra", "output_tokens": 201, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:59.991382+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5bd8bf55-ada7-4f8f-b595-11939d596d31", "result_sha256": "cee5468055ff0de5848cd8bbe46e496f818b2f4f99621c2c332cc0f7c74c39d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9661910269642249, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:08.712059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "600fa95b-a498-49a4-8757-566f173730f5", "result_sha256": "7576790edd9cb75cdf7e536159d6752662014e5597005ba9c63534d98ed73912", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0093956070486456, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:22.788186+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "482e74ee-fdb1-40bf-8e39-b46701584844", "result_sha256": "84bd957ccb887ef0007d82440337f6058498c6888830274846afd437b6472858", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00504, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.5448733889497817, "model_name": "gpt-5.6-terra", "output_tokens": 25, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:10.163386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c63bca86-d64a-4b78-92f9-5a53090baa0d", "result_sha256": "89183c8df57c96be18b6960d43fa5d9aea5f55bad36338c98aac97ee632ea9ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9825287599815056, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:41.452154+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8998cfc-24df-458a-9519-6ff29cda468a", "result_sha256": "1e20bfe8b356dd1cff5350e7e7dcd353806248e3cd7f23ebb5a70010664dd9a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9728321810252964, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:46.677620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "840fc709-e808-4a3a-a050-32a1a1c0d9f9", "result_sha256": "26d79d1696daa56a12664f45af8b039a75b3409c169bb2e78bd7d601c28911de", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9092168110655621, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:13.847919+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "782f5d63-8b80-47b6-8626-6a3d36aada66", "result_sha256": "0fb0654398036fdb16c3b3ac71258ffe8ddf379b8477b7188644e736860e79bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.6146749620093033, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:55.367092+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "cb96cc91-2151-408c-ba20-590ecd0f0114", "result_sha256": "bcf09826054b98f7cc340ed1aa48669afe1239315ebbae40888aec3bab74bba9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.090792002971284, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:59.006681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e547b321-166b-4ae0-8bce-3c09b2aff159", "result_sha256": "c00d3d412b27d390cbd946518a5703567744e046af317c29a3c00875fe152283", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0275444189319387, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:37.321003+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d123c354-0d10-4f51-8ee7-145e663c6bac", "result_sha256": "599e2227fcd425a1f550184302968d5d8f565be85dccf9398ece9eb61016d849", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0439760079607368, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:28.330341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5297534a-6eba-4b82-80af-ad6d6859d0c1", "result_sha256": "b4305e9f63d1a5edd3c0fd8b6f278a434d1e5965b63b00494717b3422b852824", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9285812089219689, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:32.795684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "bd0a61d0-b79d-4255-b9d5-986a544582c4", "result_sha256": "0256722c7a073ef39031949a13f4fa0c1d5adf7c2a5995743d9fbc6972bd51a4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.000796628999524, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:22.322928+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "908206db-cd0c-4300-ade4-cd153ebba179", "result_sha256": "9726336bfbc571235026f505c8f3f30a7788a2a92335e5fef2d3cf73d234b16f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00544, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7988968379795551, "model_name": "gpt-5.6-terra", "output_tokens": 56, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:03.667078+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0bbf39f9-8071-4a54-9fac-16453c5a5b4f", "result_sha256": "f369447b447d17fd0eb95113acbf7ac4029f980d49af56c1d1202468656b672f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005536, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7037369899917394, "model_name": "gpt-5.6-terra", "output_tokens": 64, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:29.893107+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "897fdce1-ffaa-44b9-b420-11be2fd87d6f", "result_sha256": "2905abaa7a4a607d9fac770f648b26a8bbc33a44f7cc11321b11ce191bded43b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005416, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.5169171249726787, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:11.898696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e76b43af-ec76-445d-b45d-e02c0043fe70", "result_sha256": "fec160177e4639bdcc93ddefb2f8aa189efb63c1a7a4d94398f3f7880d6edfe2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005464, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.6048626960255206, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:28.514609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6a233a25-3fbc-45b0-87d4-614a07b0f9f0", "result_sha256": "8ef46a30015742af8ae7a9f9b7f9def3bcec3c53fb7f1dbe02d6ed5fb38b2a34", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005416, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.6031076459912583, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:28.622424+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d1bf31dc-566f-420d-9380-0e38354b5227", "result_sha256": "a427afec1079f89e81791945949022178e20d4eecce4651bc88f1f3659e3e0f9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005452, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.6408740659244359, "model_name": "gpt-5.6-terra", "output_tokens": 57, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:38.774911+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f0aa05f1-b7c2-4b72-8001-90777f4c9782", "result_sha256": "26a0267c4d0f20e71a4a34a4fe85222f42bfc03adc7399d232343adb63842cce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005452, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.5035825170343742, "model_name": "gpt-5.6-terra", "output_tokens": 57, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:58.063644+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0115f6ea-029c-4aba-aa06-c344ba4aba2e", "result_sha256": "6b8b981f2aac19e86ea8245205747138e0da975163ae3e6b8c66c5bbf97e5589", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005476, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.3030235539190471, "model_name": "gpt-5.6-terra", "output_tokens": 59, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:24.808359+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a95dcdd-a6be-4c32-8722-394e4fd070de", "result_sha256": "9c52ae162eb06071f28d32bb3cdee2b37b2f15c8491b2fa2c81e1c4048f821b4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005428, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.5712887500412762, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:38.253135+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "79f45a8b-65d5-4172-80c6-1744db93584b", "result_sha256": "bc5046289540507237b3aff56a5c88db51f024dd827e435b7ce18f00491a8644", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005344, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.5986394779756665, "model_name": "gpt-5.6-terra", "output_tokens": 48, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:15.304514+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b0c631b9-c9fc-4a14-9188-b1c43bf03150", "result_sha256": "f1d44822e7c7954c9ae360931855f874c0dbdc3268360c7b08247988df14c574", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005728, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.065900538000278, "model_name": "gpt-5.6-terra", "output_tokens": 80, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:08.550406+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "88a21f02-8f01-4140-9b01-b78184cc8b5d", "result_sha256": "f799c977ea9643a3876098a013486654fe3f3825dcc7799db1f9380fc50eb24d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005572, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.064939368981868, "model_name": "gpt-5.6-terra", "output_tokens": 67, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:20.301353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8f490830-d325-4c7a-8112-a637cfa057e6", "result_sha256": "425dd24dc13f58bf1f167c079b6d1bc5d8ad0168ab57c5e0b856f03ea91cce8d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9536194039974362, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:04.535970+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b984ebdd-f055-440f-8c22-461c6ca15a18", "result_sha256": "f20de868de3eeba0f5b980938be93d8d662d8c9327df2febc5bc76b38e981e9c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2081568000139669, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:11.390153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7027b5f-aaaa-4a8f-80cf-601c7b11365f", "result_sha256": "2448b150d0ff7ecee8df586e5838595e64f67e83dafef2a9a33da87ad105a1f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.0954881970537826, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:01.742531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d245191-f1d2-4455-8505-8086ee393820", "result_sha256": "1db15718e7de345d2bd030e1d3201b57756ce043dcf3b6929d6e0259c9a161d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005088, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.239227335085161, "model_name": "gpt-5.6-terra", "output_tokens": 29, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:52.188477+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b94f579-83e0-4ba8-a578-1b07f19c84f1", "result_sha256": "3fc677ad21f30ab43da0cba6bf26f21b4968e4ae9ca77efada0f2af1efd74a4e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1806626769248396, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:31.027923+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "264bab0e-0753-4cc4-ac4e-ccd3a0ad6021", "result_sha256": "ccfb9a8d0770862be8123a5971690562d2d8ac1635c864fe15a5ca3cc4b9e950", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 0.9653506799368188, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:43.032050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "40e46d25-d884-40e3-bef6-7462c6732e1c", "result_sha256": "b6ee913146f37727aa03112167a507ffa002d715fac65f6617fab6169b583aab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.2261585230007768, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:50.853028+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "24a52037-5d1e-4b4e-8b8b-08184518893a", "result_sha256": "e5ac0377c34b4dc0620b2a5290ae16bda0438842e6e7c6834fdf4abded94ce03", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1777842449955642, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:03.840665+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4047df4b-6a4a-4827-947e-cea3d9e95e6a", "result_sha256": "a0d6aea45086f7988ed945d9ee2ad0ae966f82343be7e155ecd376dfbc1d29d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1660281899385154, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:12.628522+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "82480d72-7b2e-41b4-9771-05b86b94ce4e", "result_sha256": "adf3a6dc830c7360ad091bea7874a44d469efb8257bea3e376914d46f45fa610", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.1868378699291497, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:26.315284+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c597b3dd-39af-4180-8362-6e8ee9c1fb89", "result_sha256": "c1b813660283594b93894d662cdd9bee0136f144e8791379ac575480d8d94fe2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.3018937619635835, "model_name": "gpt-5.6-terra", "output_tokens": 30, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:51.893195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a07c699f-aa14-4b86-9679-78c9a923fc61", "result_sha256": "094d040eea12dabcc6ad11fd3d9402527f748e8b68af80407d2ae428331b0d9e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004908, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.562754544080235, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:51.402584+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "95878c21-5fca-4398-8895-51e58908bf2f", "result_sha256": "6b8bde280a9ad7527d46118c005e1e5b0536b45b6ad24fdaf4c5ccef10e1b19d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005398, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.4586686909897253, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:54.547752+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff4519d3-dd12-4a9b-88a5-58aef3836687", "result_sha256": "07f2cef845c0ed306310b1fa88e3936c2ac0727f60769ef708ff357d337a916e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005398, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.3755663969786838, "model_name": "gpt-5.6-terra", "output_tokens": 51, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:08.777531+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2aa548ec-e126-4111-a54d-41d23fc2c95f", "result_sha256": "fd573fd78fd62557bd819632a539bfaf507c1183e099ed5f219410d90dff54d5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00547, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 3.7052989000221714, "model_name": "gpt-5.6-terra", "output_tokens": 57, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:44.668951+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c77d3965-4898-46f7-a99a-06f551d7d005", "result_sha256": "ae45b69becec3cb20a1096ddc6aa576cd3deea7d128fa115a095bc629d9f7f60", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005494, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 3.166271475958638, "model_name": "gpt-5.6-terra", "output_tokens": 59, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:05.191368+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d6fba2f2-69c2-4a6a-9e0b-bbecf94d04ef", "result_sha256": "05fe7bc7233dd6e6819efc349cf6341892d9cff5e9bfb46fc0715bd6e4c16c74", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005566, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.5915325459791347, "model_name": "gpt-5.6-terra", "output_tokens": 65, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:25.341770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "287d398f-49a7-46db-8ce6-c5b30768aafe", "result_sha256": "de73a5314a87c53c1a491467d0ad7d201f69a46a0826dbbae47d480d2a8da77b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005518, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.623903839965351, "model_name": "gpt-5.6-terra", "output_tokens": 61, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:18.233912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b16a5e84-710e-4c19-a654-4b9c96b316b2", "result_sha256": "cc6b37e8b5d5e559dfc8b7b7851747def547ed68964ce8edb167019b50f7fed4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005482, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.4402815649518743, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:34.833055+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "68b58928-ad7f-4811-994a-ed1f8bd261d2", "result_sha256": "aec41e9b6b3acbede6784b436438ff3534a3dfec2ccec41c36127d899893a2b7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005494, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.4194080930901691, "model_name": "gpt-5.6-terra", "output_tokens": 59, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:39.820096+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac15f4c9-44d5-4227-ae3e-e649dd75d364", "result_sha256": "c5798697068a960cf9a4ccc3cfd6d72b950489a757a8175286544cb5e6cc52e2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005194, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.8016148869646713, "model_name": "gpt-5.6-terra", "output_tokens": 34, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:56.796999+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "81226e7b-a3fa-4245-ac0a-515987d97be4", "result_sha256": "b4d1f1b3d2faa743d72fe2e05654cfbc81629c73de5e78cf9ecf0390f012f16c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005458, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.7601991120027378, "model_name": "gpt-5.6-terra", "output_tokens": 56, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:05.856991+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a9319b99-3ac5-4767-807d-56139401c1a5", "result_sha256": "90fe2ce8875867d4615fb22ac02e8ada4f885fd6a7c9733c02424d3847ac4e5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005326, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.354639911907725, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:27.122962+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9b91bac8-3b9d-4fbf-8d2c-3ac9e76ad073", "result_sha256": "f1cd36e8c85c7db497a5aa05a7ba788f002b09d1db9fa90592db72b391e39397", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005266, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.4037042480194941, "model_name": "gpt-5.6-terra", "output_tokens": 40, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:16.417194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "03f01b6c-992e-4456-8dec-908d5e28f6d2", "result_sha256": "90868a2b974fa301b8984a529f7fadb79157b583748211ec092a4207cafbf01a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00544, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.4448895490495488, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:43.042922+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f6014e8d-98d2-4d80-8306-b253cdef1bb2", "result_sha256": "0ba17fc99e42f26555004a6efee53b603e40cece07dc228c1685ad6bc61cf855", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005476, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6088955219602212, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:14.723663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "75457887-4c6a-4095-ad09-99d5050f0dd0", "result_sha256": "b4b2fd3f7e63d8f7fb1f2ee19dd7c0018805f19db5b694ae353938e4907b000f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005428, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2296600160188973, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:27.865089+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d4eea988-4fb8-47dd-9d49-802299dcf873", "result_sha256": "4542648b6dff45398fa3844204dea4c1bcaf1ac59133fd4d40cf912f54859ce4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005428, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7608591540483758, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:10.825085+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "64984d5f-c364-4b3c-b6fd-584549f93f1b", "result_sha256": "a60610209e9e4311b043296ef6ab0e6760787ba469b30f8af2b51a2707174afa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0460967799881473, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:06.651350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8c48656-ea59-4e93-8dc1-089f2ff09c25", "result_sha256": "928442d4614d249563b0bf074bbc3dc3cd3a6135a4af64e73adf3af16323823e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0778363180579618, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:20.012826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5bd561e-ccc3-441e-9d49-d83b031dfe10", "result_sha256": "25155e9e33551509f8e7e6caf9c1dc003055fe8f214056bd7c645b8d4385242c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005248, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3710415579844266, "model_name": "gpt-5.6-terra", "output_tokens": 39, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:42.527177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "638ba0e7-b3b5-4f5d-9f99-00087087b289", "result_sha256": "0466463a6ca70a975713d5405472f622d3427f684dd80eeec4c433ab63e4db78", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005332, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.3936986789340153, "model_name": "gpt-5.6-terra", "output_tokens": 46, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:30.017903+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "490f8690-b44f-4dd6-823b-fc4a851e037b", "result_sha256": "dd6e445a0c9dc1527661e25acf9c10df159ade094f283f2e63b1c377e0c6370c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005224, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.346108243917115, "model_name": "gpt-5.6-terra", "output_tokens": 37, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:46.273542+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9f94f1a7-5602-44a6-8ece-d8f01676df02", "result_sha256": "59c5ccfea477f1f9e2dc2afa8e4d752f50fb868097217535cd015065566f3d45", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2471144960727543, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:22.124342+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "838f7a7f-719a-4fa1-871c-6e492f58c9d3", "result_sha256": "e567879d9c2a2ac67185653931feccfbc1b2886f02a985b35532af3fe35bc85b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.085568096023053, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:22.741942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c28d5cf5-dd5b-4653-881b-99e9f2991acf", "result_sha256": "13b7bdc4778095bc2932cd7e74f94e125079ba89566149e5a253c70d9f4543ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0052, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2198352189734578, "model_name": "gpt-5.6-terra", "output_tokens": 35, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:43.916966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3aedf5d7-7370-4d87-914e-cc68b824e7ad", "result_sha256": "2bcb84982f6075e932f218a9645be8d0d864f2ad00881797d718146a0a349d87", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005624, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.3746887859888375, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:22.905902+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d6cc9d56-6e60-4157-a2cf-f6574ba6d096", "result_sha256": "9eaa0321d4511b43d67decea42c267e45e3efee359075ab9e351aa58f7c545ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005636, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.7279986820649356, "model_name": "gpt-5.6-terra", "output_tokens": 69, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:04.896235+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "022d0ca3-d698-4a41-818d-8047411e30c6", "result_sha256": "533a758b77274b5c91a56915de15c8c4e75bc5ab35f3ba1125069f1c45444d6e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005624, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.9072029379894957, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:22.900304+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f16db085-9ae4-46b5-96ec-75585541ccc1", "result_sha256": "3b391011f1c43f69fc42465e8c6a3d906f9a2f7a53cf2b5dbb3b45a694f3773c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005564, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.534426738973707, "model_name": "gpt-5.6-terra", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:38.608232+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "dc5e70f7-fb55-40b4-a3b7-c079a87e953f", "result_sha256": "bf224908829040762e9f7812929022586c29708008bb5f09b85e3dde711fdd62", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00566, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.7702140850014985, "model_name": "gpt-5.6-terra", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:46.485173+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3ca87ebb-9486-4b66-abff-1fb6bf34b642", "result_sha256": "08b3d25a4fb433e1ec9d25772d225e6224d66b14049da3a934fbfa67bc6389aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005672, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.5688131119823083, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:10.081492+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ef360fac-9ef9-4a0d-95d7-ce0f43a86c4c", "result_sha256": "4076347dfef075374144d803a1aa02dbbc917ce311e6ea2533a5bcf2b1c2ba51", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005924, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.088609583908692, "model_name": "gpt-5.6-terra", "output_tokens": 93, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:43.730060+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "db030287-4ebe-4f80-8825-e2a6b7e3f58e", "result_sha256": "1eb25f3adcb06eceda2e5ad0c2dde4664b18a42e06044c9b177ec8d462fd2af9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005996, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.404841198003851, "model_name": "gpt-5.6-terra", "output_tokens": 99, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:38.942806+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "64bb0746-b36c-4804-8a1b-8c1f9a27085e", "result_sha256": "2e7f6d364d6d1f45b4b05d4ea78bbe0681c8c22131cfc778f40758f3d6690c48", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005648, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 6.537707163952291, "model_name": "gpt-5.6-terra", "output_tokens": 70, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:04.768875+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa7cb16b-741c-4961-b3ea-ab386d18c099", "result_sha256": "ea4692e7c46366b12fe5785056952311f8bc899636fdb53fe3f51a3d4aef06ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005624, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.5527777209645137, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:12.471874+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "882a5508-8e48-4862-9275-221823b4f270", "result_sha256": "6f1d0dc918dc0a0472360bdd6c20999a6d64bd3d8aac328465c36d1908ae0b0b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00572, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.065890035009943, "model_name": "gpt-5.6-terra", "output_tokens": 76, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:18.156011+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3aad628c-9bfa-4042-9e42-186cb569ef36", "result_sha256": "f721c63dda2d3eb5720b95396f711f58d01542df08c56cd8d6b6a5cba6e5f091", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005624, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.9622918310342357, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:36.167492+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3566c82a-2fa3-4b8d-87ba-1393364c485c", "result_sha256": "73cbbab3039497f93d019cdba3a403fe27684f3492a5063fefea920a984537c3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0446943810675293, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:53.900785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2bf6b5ea-1161-4626-a7ac-7aa2c9fc5b65", "result_sha256": "a4c74de5397d74ef8e3f3e5ad009e4e5eca0a3e3a5a1bcccf794d249b7eba8dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0440604130271822, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:00.256598+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "bd36422b-a8af-4562-a8b6-debb5325a7b4", "result_sha256": "25a815f32b591a844978e8368c86e34a047e13b775b7715ef913ba5173d0a0e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0580253720982, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:22.010408+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "44ee487d-8655-4ab1-9f90-b5392548f50f", "result_sha256": "8a6c96cb6b732cf65aeaf4b79c0d77a6e0225541d45896d2cb6a494da0b5d022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005224, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6069251610897481, "model_name": "gpt-5.6-terra", "output_tokens": 37, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:54.677843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a58ac3c0-1b66-4cef-a278-5d1b66f36993", "result_sha256": "00f10d84422bff1afd1b52075e1a63d6033a60362a7a94f52f3aadb76a59b07a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0052, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2980983939487487, "model_name": "gpt-5.6-terra", "output_tokens": 35, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:47.100848+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2cc515c4-7a3f-4ed2-a03a-fb00f197c4cf", "result_sha256": "5367661a1d5b7a47045b9df99283700c3357f14c3576485280d775ff93ef8687", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9687036889372393, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:20.182497+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "21ec074f-0d99-45a8-bb9f-fd43c91f9897", "result_sha256": "06eb94edd451b77c7a7a3c7d83cd2b8651e81c60761374c7a19d4107b727d3df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0538448240840808, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:26.635707+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7d80d2cc-f65b-4de8-a78b-8db78e647bdf", "result_sha256": "e7b3594bc3a1e007beb7b95f5e7f53dd19e1cfc456567d278bfe2fef838b5e46", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.10583849507384, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:45.817941+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "fd09509e-d4c2-43f8-8c19-76ee9b7c75b3", "result_sha256": "2cbeb37bdc985efe3c9e136ace43eed81e05c11bf044305ab9678a959c01c59a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0880251029739156, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:09.151424+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "95305ffa-0360-4d39-b6f5-446d89b82ce3", "result_sha256": "d8b6acbdc3d1b83f056c6654477e13116d66a8301c4f171c3804349bdbf53a87", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1550613860599697, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:12.542313+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "87630261-bd8a-4564-9d31-012ffae21a98", "result_sha256": "29dd1790bbd8990dbda3e5c99ac9948b1d16527130103554d6c2d1d438429bb5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00508, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0438787839375436, "model_name": "gpt-5.6-terra", "output_tokens": 25, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:33.538334+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4db30288-efae-41ce-b0e7-b1d19f73b32f", "result_sha256": "80715790d384d507b9eb363728fce7cc7cf0a6a32e34426143aec1ab9a20e84a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0593727179802954, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:11.867472+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "eec0e4f8-a528-43cc-99b2-d7b77314df90", "result_sha256": "4cc94ef2f12c4dc0227975df3898f8ea852dfd65b53007cb2da040c8b4648c39", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0459219799377024, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:45.052619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a18dae2-c6e9-4565-b9fc-897f1e06339f", "result_sha256": "1cee73451164f3490bf30b7aac5e934237083284ff0bc8fc8a42b0412d0ea8ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9731998980278149, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:33.150957+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2604ef45-f9e0-435a-85e4-7f56bf3de293", "result_sha256": "1415b28fe4e48b23eea346e11e98cef4ec994508bc915295f01cf492681eb3e6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.9789502540370449, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:37.719929+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6e662405-222d-40e8-be12-1754bc14f026", "result_sha256": "52ffbf2a845edb71d6d061b622c407e0a1a34174c95c45314143429c4c55849d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 0.99567639909219, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:33.030671+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4a49e9d-d094-4fa2-b611-ce13e5ecde0d", "result_sha256": "2a40ae6613507092888bfb356dd9c065c51e5644fe908f655925e2262ffb47c7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0571022579679266, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:12.111425+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e84502e-a1d9-4ad2-ab0e-96808ec57d1a", "result_sha256": "88c2c7ec9330c8a0de2738ba6076358c1436d1e0141773c8741f535d6d730242", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.011452489066869, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:14.669071+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6cc1c5ff-338f-4ed9-ae57-5ceb17ca5e54", "result_sha256": "9de8ca21590d12b73e4ec2c2072d2c46d268bc058995378ba1e30eb095b7921a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.047896070056595, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:53.975071+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a43252b-98df-40da-9e24-2783fb1c5ada", "result_sha256": "06978abb6e0714d589318664ecc331d08762cc4e44281e13877cb644e68553dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00508, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.35691913892515, "model_name": "gpt-5.6-terra", "output_tokens": 25, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:46.404568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2ca5ffbf-d091-45eb-8339-724d2df77e3b", "result_sha256": "17404702c796f7c80fb3860b18a5b9e3cc27e0e1f800527060f302edf159a1f2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005068, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.4410297069698572, "model_name": "gpt-5.6-terra", "output_tokens": 24, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:29.353867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a7f3c027-7843-456f-843e-6dfaf39f9e90", "result_sha256": "2e47942325a7d8971c29034ac0279ffa5ddf39d1d3e777d96582171457eb6dd9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.1851296430686489, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:05.295803+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d335d2b-3564-4f96-b888-b240992ae02c", "result_sha256": "a35b763a1efe901961da8fb8853f73b85d4d308f26cb63cc029bdae3b8645ecc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.0394072469789535, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:37.907379+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "12c15b7c-a66f-49e7-ad53-a0712ae47a72", "result_sha256": "8002b784e6c62fc62dab361a3ce65f299ad8cfcb445c00ad9efc85b35ec9e49a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004948, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.2328114500269294, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:06.097620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "047f1adb-04aa-4225-a6aa-057f02c90c54", "result_sha256": "c0d01970e0ac36c1785f941a65837b37deb15a3ffd47e4d25366db766c41b248", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00534, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.5432474220870063, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:22.940817+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2181c2da-ac1c-48eb-afdc-d963b427243a", "result_sha256": "f0d2bc60ec8952f7c40ed747d232661eab05fde24c7b1e4d47bfa4430c8d2366", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005184, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 0.9955424609361216, "model_name": "gpt-5.6-terra", "output_tokens": 32, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:28.951945+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8687bdf1-8357-41ce-9c32-2df56ac6baf8", "result_sha256": "5e1cc0a47b1df5856cc4dcdef708c441aa42e8653b11d94d5caa76c4ea7994f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005496, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.813144402927719, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:30.346016+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "931b2dfc-8bfe-4715-be98-af6ee267f639", "result_sha256": "41259a933237de1afcb58c276ecb80e903a94764080842a7da68e2b248a5da4a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005556, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.050975520047359, "model_name": "gpt-5.6-terra", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:21.653484+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "90bc9051-bb0e-43b4-aecd-c6dafc98c2ed", "result_sha256": "c9ba6d3a3f840d1c1ca2d29e00408a23f82ca2b80fc010d2be964b9e579a2154", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00528, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9320375339593738, "model_name": "gpt-5.6-terra", "output_tokens": 40, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:03.650337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "83001e80-a75e-4456-9bee-3412899b7b92", "result_sha256": "d2ba7e02230dea3a96632bcba0ff2a4d28dc2f41ccd79c66639f6728e6648d9a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005352, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.430087040993385, "model_name": "gpt-5.6-terra", "output_tokens": 46, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:03.777837+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "450ba488-4a25-44dd-b6fa-d4458f9a0150", "result_sha256": "33cb83d200eb673173c3f60d4b10dbebbeb9417689e976b749f744bec9ff8df3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.72255479299929, "model_name": "gpt-5.6-terra", "output_tokens": 53, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:34.777692+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7c4540e-c021-485c-a252-145d56a2abd4", "result_sha256": "cbfd18e2521754c6df0ac8acc0c7e5342c54d21d13510c978d04cedef6ab30af", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005436, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.4238748570205644, "model_name": "gpt-5.6-terra", "output_tokens": 53, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:23.966876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "57266cad-d994-46c9-b381-8659750438a8", "result_sha256": "e429cd7724eb5d06754d1fd67bb85126dcaae47832ef261c953b6dddd974f181", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.3671110660070553, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:04.032900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "73c871cf-b7d9-4d2d-8256-63ea0adb9c7c", "result_sha256": "0e5862d880f46fb0e450f40e2f367b991bf5ce06a2bfe8b6e56123039726cbb0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005652, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.845561953028664, "model_name": "gpt-5.6-terra", "output_tokens": 71, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:29.022156+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e1e91dcd-c74f-4ebd-8437-f9b4b9491488", "result_sha256": "d3eb2222420fb0dbefe3e08f70305e1f3aa3bf22e5e4c51edc6560b5c20a03da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005856, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.76601202506572, "model_name": "gpt-5.6-terra", "output_tokens": 88, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:18.570770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2ca8820c-4548-4d77-936a-404da63b65f4", "result_sha256": "5251df815e56395663cf4a37b4bb30343d6d50a5b7384d2788cc9940a3d171f9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005496, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6598255980061367, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:44.130816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "315116b1-0920-437a-84d6-e40f33d23118", "result_sha256": "04642b502b4cdc77a19f0745b5d6b1fc27ff3987076c1b9daa329f41c5ebb105", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005676, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.0356965459650382, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:19.057815+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "69ac9cd4-228f-479e-a85b-c88c3d824006", "result_sha256": "894fdc1e79f49e58f49092301039a9b7fe8fb70755b51c93c9dd9437f70ffa62", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0057, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.8561351000098512, "model_name": "gpt-5.6-terra", "output_tokens": 75, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:19.836487+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c9b6135c-0c89-41fd-bc69-fa991ef1c3da", "result_sha256": "9a0f840926f224f5d43a941bfdb878858b0e6ef0628efd5c9d5e475a67a7eb8e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005556, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7237675728974864, "model_name": "gpt-5.6-terra", "output_tokens": 63, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:36.513006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "42073d74-9ab0-43fc-9d15-070364029b64", "result_sha256": "7606bf3358ad8d0c71e9e85f79aafb6335d1ef328cbafa73f62334af24a7098e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005664, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.563726330990903, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:07.650163+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1122b7ac-25cb-4ddf-a5b7-03fb0797ee98", "result_sha256": "fea1e5202a7bc4c4017b679355daa8d424122df20705d6b10878aee4fcbc3ff4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00546, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.458061649929732, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:22.470555+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a88c12d0-d276-4a67-bf8b-82f49b1da66f", "result_sha256": "02702a88cb8336457abcc66a313e22eeeccc8e3d7ddde5a670f1d43eac6d1f93", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005592, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.1344363620737568, "model_name": "gpt-5.6-terra", "output_tokens": 66, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:32.068947+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "dee08b51-5522-4ce3-989b-ba3b4ad06ad4", "result_sha256": "8d66bf2d13d24b4446e257bb7290c77be0213b7e22dce1ece16187b3c901da47", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00546, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6617704869713634, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:24.431378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "332dd20d-b1f5-4934-a37c-3fc4f31b1d0a", "result_sha256": "dbb1ca8972c6ccde985e74819ddafa6ab3987252d1fa0664c395ab19c98269bd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005448, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.86307211604435, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:12.278938+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a7aa253-9c6d-45be-a322-a03d9ecec59a", "result_sha256": "d663ed423e0565b08a1eb6e10fac0a767000fb835d3b0c5e70e6a7b449797de8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005532, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.5869403240503743, "model_name": "gpt-5.6-terra", "output_tokens": 61, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:47.470143+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7d5ec978-ba82-49a3-8d88-3a7760cb1756", "result_sha256": "694eddefeda58d8e57f89f9e0a6c0d78715c804f6378a14ccb374bc6781779c3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005616, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9577694750623778, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:58.768677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a16a411a-b5fb-4632-a271-6af9fff5744c", "result_sha256": "e3d0f735b46d55b766e4852d678037ad388f6f4cbf827a318655d04aa9b936e4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005952, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.0367783510591835, "model_name": "gpt-5.6-terra", "output_tokens": 96, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:27.767682+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8b183878-7c44-4841-b58b-94dd49862cef", "result_sha256": "73db51333b65759615ad9780eef86ed12e58b0a3c05eb6bc5d426600de87598f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005724, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.65151071397122, "model_name": "gpt-5.6-terra", "output_tokens": 77, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:49.422691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "05ffa803-262d-49b1-b0bf-803cdd46e545", "result_sha256": "9831ef2efce757d198ee6e210a0385183e780978d72c6ab327314f7af16cfb9f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005692, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.1393535879906267, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:58.315981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "20b2eaf0-eea0-4319-be0d-d066830989b0", "result_sha256": "c091b80f7fc5c18eec59eb1601c45d9466d11813b70b40ce4ab650470f010bbd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00562, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.7139569050632417, "model_name": "gpt-5.6-terra", "output_tokens": 66, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:19.357519+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9332c05f-f1fd-4dcf-93e0-9b8d2660de80", "result_sha256": "fcfb62ee2ebe31fbb0070a9b71f660f31c6ddcc5cfb54df13dd67c52f4ec7b82", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005668, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.2154372490476817, "model_name": "gpt-5.6-terra", "output_tokens": 70, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:12.115680+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a28b0746-638b-4042-944c-92a1b7d7ad21", "result_sha256": "e0bd121611b10bed8db0e472967ab5a3aadf46222b94ab13d2b8d2d0749a1cba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005644, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.9501131169963628, "model_name": "gpt-5.6-terra", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:47.043665+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "def7e569-aca2-48dc-8156-5d39f70b0f22", "result_sha256": "b06ac08036c5f2f0c9bfa34fa7689479e8d95f0b6e1bcd0eea0c9773f5122a02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00562, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6463626580080017, "model_name": "gpt-5.6-terra", "output_tokens": 66, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:27.209064+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "01d3a111-9a64-4ba3-83ef-80cbe8ad0bf5", "result_sha256": "c1d1ad737134ed827c193a1fe600d8d3364e2a127867c36b10cd923601ad4ea3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005692, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.5033974880352616, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:31.752513+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1317664f-8978-4543-86db-7d07fae5d18a", "result_sha256": "aa0ab2d995ebae89db1b7832e3aa62880b6e6aadbc85cd38a0fee46a1909d23c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005704, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6409634209703654, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:13.419485+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8276108b-2ab5-4c5f-a4db-3e378c60fb4a", "result_sha256": "9f293ccd5f6b078b3459138d11c547474e65705ca613030f15bd16d3da592873", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005752, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.8353930540615693, "model_name": "gpt-5.6-terra", "output_tokens": 77, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:43.746888+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ddcd476d-c759-477b-a2d2-7274db782792", "result_sha256": "639ea633e7a85292dbd6d0eeaa0a4a5539a6a71f0af05cf8b376de418ea67bd1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00556, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.8581002879654989, "model_name": "gpt-5.6-terra", "output_tokens": 61, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:38.109700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ab016a9e-bf83-46f2-88c1-1f791838becb", "result_sha256": "a58fa7e98b69ac9d540e60a1b9bb69e2635f22aa96e8b1747fb994da74319fa2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005692, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.0180877830134705, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:47.407041+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce5b29a3-9ca6-40ef-beeb-123014ae5f96", "result_sha256": "503f5cc112790a39e67c9f97c0dda9728c427dfd0bec5b5ffeb2a318fad2136b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005704, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 4.1431485519278795, "model_name": "gpt-5.6-terra", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:35.559927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "03aae3c5-7bd6-4d82-b2fc-70482b4c3879", "result_sha256": "931382b1b2832d3f5048896e50dae3e3b53e22a96f4b5f596b44445e48ec2115", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005908, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.8431205149972811, "model_name": "gpt-5.6-terra", "output_tokens": 90, "prediction": { "choice": "A" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:01.285954+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "355d12ab-8b71-4397-acd9-8e9c53e12aff", "result_sha256": "330f2ffbfd71887532ff525a66433e0da3984a3978856ee459fecbc11ae230d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005628, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4257578930119053, "model_name": "gpt-5.6-terra", "output_tokens": 66, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:25:50.849564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "932caa97-50de-46b0-9f0a-5920660620b0", "result_sha256": "93434c3eb6212fa3bdc55e2f00ca696ff02868dc3e452bd04d438e61e8145340", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005376, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4385217500384897, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:47.482988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b22a3715-fc8e-4473-9920-9be855ff3ef0", "result_sha256": "9473b6be2e26c4645f136137f96fbf87ee207c8dc8408298bd9de78d87244559", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0066, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.2920627129497007, "model_name": "gpt-5.6-terra", "output_tokens": 147, "prediction": { "choice": "D" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:12:48.871312+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "41ff4d85-bc12-4943-b6b7-31f1c72a0708", "result_sha256": "c153152e9515e60eb3111ca41e208642c3014808282222f80766dfe68bc44b8a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006012, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.077666606986895, "model_name": "gpt-5.6-terra", "output_tokens": 98, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:15.287349+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ace6795-bd4d-40d5-80aa-227526fc7059", "result_sha256": "612eed2359461fe7ce23c63ab0c8d220ddc54e20356d71f43b175110e51e589b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005952, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.0404165079817176, "model_name": "gpt-5.6-terra", "output_tokens": 93, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:52.254821+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b8de10e-3d98-4ce8-9f48-827739f2f63d", "result_sha256": "1541e0c16282596099d65eba29dac6139f2d9ca9138989a43a6c23e5864169dc", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005532, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.7908517440082505, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:39.376526+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9badc219-0a4e-4b4d-8fbe-825bcac7b219", "result_sha256": "6f73fd1f1b0c108a383941272d472165010707a42b7fb354cd3b820b4f598689", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00528, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.3667318450752646, "model_name": "gpt-5.6-terra", "output_tokens": 37, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:14.368153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "18036ed6-e416-410e-9f09-cf99daf5da13", "result_sha256": "e0f9a947c007f6a5391ee0aa482e5de619d529202ab5443e233bee3d11064629", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006684, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.918843007995747, "model_name": "gpt-5.6-terra", "output_tokens": 154, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:40.732632+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3bcbf9ab-ca34-4699-bf74-54f9cdcfba87", "result_sha256": "56c1ff7ee1a7e9fedc3a54a19fe7235129eb5b62f417252ba8da47cf1109e3d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007836, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 4.791155445971526, "model_name": "gpt-5.6-terra", "output_tokens": 250, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:24.623093+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e4d9b079-d0dc-43b9-956c-bb2e97bfdec2", "result_sha256": "09ffef2b431d00dbdf2d262a8b3ae8479f697f52b3c5bdb30405406316e4a1bb", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00534, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4324233559891582, "model_name": "gpt-5.6-terra", "output_tokens": 42, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:53.817653+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d5267d95-29ae-49fb-878e-12b4850cfbab", "result_sha256": "531dbabb8c9a6513429a746f2fe8b89d4c34ffc2826130d2e8ffd12e4ba87020", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005292, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4787304729688913, "model_name": "gpt-5.6-terra", "output_tokens": 38, "prediction": { "choice": "D" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:13:02.863437+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "51afe80f-986d-4d6b-a00c-c364b8b59ade", "result_sha256": "95c555983bb4ce096b126a5b35e2159050e5e2fb9bb8633e2877076739cba4a1", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.005364, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.4801565950037912, "model_name": "gpt-5.6-terra", "output_tokens": 44, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:29.055055+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f76d91ab-81e0-4641-91ac-9b3294d8202e", "result_sha256": "d81f74e0e8fba096f7ee1f0e79fa2f20a7e4a81cc82a664b11cfc125ec31ec41", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006144, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.761166985030286, "model_name": "gpt-5.6-terra", "output_tokens": 109, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:07.381717+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "47da524c-25c1-4223-a801-3b620986cae4", "result_sha256": "455757d482d5c6d88ff97769273af20220d1c31c5d0caa54346cc78837e9e82a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007116, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.983606338966638, "model_name": "gpt-5.6-terra", "output_tokens": 190, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:10.504052+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "971f13e2-d9df-479d-a46d-e7486cefe444", "result_sha256": "b29edbaafb71dc07578b0d820d2a47fa7c050464dc01f876e0c0f63b7e941cea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006012, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.228157651028596, "model_name": "gpt-5.6-terra", "output_tokens": 98, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:39.574350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "dcf8480f-2f92-4889-9bf1-f86d2661526d", "result_sha256": "282b29d6177908811f68df7143aab3b7cba1e923a14602aea1d1413a92e0a873", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0057, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.883616484934464, "model_name": "gpt-5.6-terra", "output_tokens": 72, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:12:58.965131+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "265252f0-feb4-4813-8083-3a4835aca84a", "result_sha256": "030d6c30a76f842685445a36653e5caece339697b8d06f39d9651b1ffd7c4b45", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.0261568380519748, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:51.598670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "952894dd-5c71-4148-a5ca-4fab4c0d9ce5", "result_sha256": "04b7cacafe750dd58ccb354c1e1af5ed1179bf812a7907feef138edb0782d74f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.0808280620258301, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:30.374929+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b39a8f23-f53b-4834-ad0c-3888414438b3", "result_sha256": "bd41b943d68360bb6d8628dd3f28a6d0c95072cdfac1fa1190d820a9f6e65b60", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.13198275398463, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:16.938843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff9ef0cd-bb37-422f-8c7a-601168648529", "result_sha256": "cf744355ee7cb06f813460bc0b99cf5e320ceba67def0c41f7fcfc583a6cbef4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 0.8947736809495836, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:26.331246+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6f4791b2-81a8-4a51-b541-b2e26db7bde8", "result_sha256": "91bb54f65a59441798589a7a91fdede4309808d16d657a4f109c84736b787c01", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1658582909731194, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:25.477164+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3373ebf-8d28-4719-8a46-d87f8d54fd9b", "result_sha256": "4eb501e9614529738a4138c452f9ea4f36b4c2d27317b4a3518d1d4a069cd8e7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.0373006330337375, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:32.318619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7715390e-d5f5-451f-9a61-b83254def0fe", "result_sha256": "025041101cf750a81ee76da552c9e955626ab8c18f8b4ab6e826659dd1c94516", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.404059205087833, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:07.624193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3369c081-e89d-4009-aed8-8ccb53e0b8f8", "result_sha256": "a5b155003be7b24fb0cfe7744621e52f89972a1409f275c63e587464ea2324c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1972656630678102, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:47.694840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "27d1d0fe-d2b1-4a93-8657-69b1bb87c494", "result_sha256": "069a82ca5a4f9548ed770d4c05fc627552d8d58379495b147aff233849e35583", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 0.9121580169303343, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:59.289715+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f2684cf9-43e2-4b5f-9eb2-e0616369d73a", "result_sha256": "b821e74cc79874d75892a0d802a429025c8d3cdec30ffd1acc77a90d923844c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.014358806074597, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.033842+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "abc9908e-4dc5-4719-a71b-bc2e7b90d972", "result_sha256": "e2386a8b6e99a6ede9e85638edee8c744b62915d655a605f742b5ee4c2d1abc8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.264705620938912, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:47.247480+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e013d857-518d-4b7c-aec2-da27ec0622ec", "result_sha256": "cc66c84beea9f3831ebca19be35061ca276124ea7c20deb8878dabb55af4e884", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005032, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.1263177009532228, "model_name": "gpt-5.6-terra", "output_tokens": 25, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:57.549470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d491b9e1-a70e-4557-a516-794873b318f9", "result_sha256": "2d2f53c19d0ebd9a6774fe3efd9221a0bd997589685c0f2d80286ba5296598c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0235399330267683, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:48.374020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f7d050e9-1294-4219-a819-3afde1318dc5", "result_sha256": "262ff60551aaf7065650a16c39d2602bbf9028df45545b8422071fa12b6582c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1204570269910619, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:13.549844+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0d716ed9-5444-40fe-be66-88bf30ef1c90", "result_sha256": "f3b65363988438669b4bbe538e9f1dff2475f2fb13a874baec04934d4d34ba06", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.6086741089820862, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:04.143609+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d6384876-fad6-4199-af8a-efd3e5a8d172", "result_sha256": "c123dcf99aff80f0d49557e1a354c354adb6c1fdf53eb8262d1414bdb96b60c5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9907499901019037, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:07.991882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ec285d4b-d44f-4d15-9052-768d2e9bf348", "result_sha256": "85794e8b9a92b5ac927c88745a029c910a749cd72e4e34d22089cece5a7ada52", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.3061421690508723, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:20.640495+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "42326d8d-524a-4817-bff8-57ff22588a63", "result_sha256": "43ad85123d63af2d5e2ce692be16c2f536109ede7026fbbca991e5df43f6f3a3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0009053819812834, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:12.442796+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9743896c-0388-4ce1-866c-8c1f74799322", "result_sha256": "92584c5b982c5a4beabe2b4ad7cd7b41a7e42bdbddaa4c3cc686602d63788f7f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1394957499578595, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:40.724548+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f07f4833-249a-455f-bbf2-6d0a72cfb6a6", "result_sha256": "57c17aeb882d48a5a0f684ff96bd0895c05b6b2e14d2f06b8fc5e32f31539874", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0873682150850073, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:50.896389+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a9099f8-8d58-4d0a-9c89-fe8ac9b2301e", "result_sha256": "c8216e8940f3a5231e620273ffcde176ef793d866de0278cbd5a5f808d54cfc2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0215230480534956, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:24.465725+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "014a2175-1477-42ac-88c6-74f6462e5948", "result_sha256": "e3b28f6a5ae37a86e6ced7d7136e4c2e3160610140b1ce29af568d372347b43b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.4974683240288869, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:26.761468+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6c0c2bc1-dcce-417e-87f6-38f3df1526e2", "result_sha256": "071dc4393c7bde229d9c1d6ad89f3b79861422733af4083418c60da8cc008a09", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.3832455329829827, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:51.232178+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0af447d-59ec-4858-b6a5-7d8ade0c720d", "result_sha256": "bfdde4a0f3af87aaeee98149862efad951e5535911b977e2d5881740fedb335a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1145358890062198, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:36.674773+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3381b685-fb64-461c-bf9e-2f53241d1cea", "result_sha256": "e449bedf0418460cadab02654027acee8537d78887f2f600c09fd587d7dbbb56", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.840896283974871, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:09.048603+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2cebf64f-ab25-4565-aed0-db401ac4406e", "result_sha256": "5e45b5859c4208990a45d590b639bf76b0231102fa41643c7b093c347fc0ab69", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.3819021939998493, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:36.787831+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "06aa6916-2f1e-44ef-82eb-926d658eb686", "result_sha256": "c4155c23f2df77f459a6758c7d58d1e4cd8a0aaca177a43e7cc92995cb2d0d9a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1979460819857195, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:07.568174+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "77dcf7e5-c150-467a-9bb3-de438dacb8cb", "result_sha256": "c60d122c2950c2c271d8a1ec7679cc66075b8a00246627283083e52083678582", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.1168239449616522, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:17.842151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "624b604b-5370-4c9c-b251-a66cb74f1f56", "result_sha256": "2b42416c3cefabfdbd3af125315e0755271b6dfe5db02ca849461f3e51803e77", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9126166520873085, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:16.910874+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "45cffcd1-bf86-4f20-abe8-42415c047683", "result_sha256": "e1cab5fb5a32dd865d593932ad45d9319a758f7b372acb744d24ccd259fe9cca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.167948403977789, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:43.094760+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7ab32182-15fd-4b28-93c3-6d326ab936f9", "result_sha256": "8bae6e28bbbe04340a12af71a5357fad4379a08cda8c0625c69f5b148bdcce18", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0404409440234303, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:55.506903+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "40f61743-e97a-451d-9231-ef1d8ab17ad7", "result_sha256": "d29ebafdc7549d1ebf9ddaae6a7081c10ecb9ba3fcc77bc9bf73ccc3ba1ec775", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9666324650170282, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:14.487658+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a42d4136-269d-4bbd-b11a-b066d8e592f3", "result_sha256": "53baf579ee39b5ec9a6a75909f343b52ed84f739eccdf9824789f8456f61b8ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0769307899754494, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:20.694930+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "cbb05513-ac92-47d6-8bf5-afb75e918ca0", "result_sha256": "de28b355ce5743a91ea951af6fa61ae66a37121408ab3463cf9355cdbeabacf6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.0905593580100685, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:36.577946+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "43b76b64-1506-4488-8651-b20b817b4976", "result_sha256": "8f474343452771206cad95f6039b7082d214530196eef9f8e59bb227dc5f5137", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.1357052329694852, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:27.151690+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "79e9135c-ad90-4519-b99b-4d4658659ff5", "result_sha256": "0c48ea3b8d4850412137b42d1bd7c50359691ca52868cbf1714d8c1eae8f8746", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004942, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 0.9728773260721937, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:23.682657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d156bb2e-e760-4727-b98a-d71b46cb42ad", "result_sha256": "315dae3fdd7de924de21e09a42f1382f8d558fd5a0bec987922711aad4cd5dbb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005096, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.197528864024207, "model_name": "gpt-5.6-terra", "output_tokens": 28, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:35.896536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "37eed5e2-9a4d-4b05-8c50-a95d842fc0ed", "result_sha256": "0d694d2f556f1f1639b07d466308ea3ea222bad7fb5dae01637093b8377f6ba8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005288, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.7199146290076897, "model_name": "gpt-5.6-terra", "output_tokens": 44, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:42.791295+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a9627db5-10c5-4da7-8fea-dc42e280019c", "result_sha256": "cbbf1202da2a99245348ef8823854f9407c0329a84b2ccd11f81557630d0cf7c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.0297149729449302, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:01.197688+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "676d6b80-a6b9-4d34-b314-dc271b9d1ce0", "result_sha256": "ab165bc82f2c67766ca889f2700ce8b72e5654eede75652f1b464d0e99eedbf8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004928, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 0.9894432399887592, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:02.193139+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ee7ec416-7b7e-4c19-9838-20e903b8e4dd", "result_sha256": "ca27180b1dfe9375b1086e39e070fead2823ca1289ba6803faa06e060b53da5f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0053, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.3813212929526344, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:14.877489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "28237990-84dc-4956-86cc-c97412dc1cde", "result_sha256": "cfed5f8a2dc4a14782b9824994796a8a95e29a219c0370dc88ee317661844358", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005276, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.7679162779822946, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:20.793035+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "fa1b660a-adec-4442-8df7-fb60da20950e", "result_sha256": "529e5e1feb114bdba6fb8a606def92666b6657fbd14b48d179016cfcd5c0806f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0053, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.3304282169556245, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:01.754241+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0b183e2-224e-491e-95c0-f9d237dbc874", "result_sha256": "7c7c9cc63f3d3e22672684d66f0fb8f4144c0513fab665a4d90b516b86501e3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005288, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.3855339279398322, "model_name": "gpt-5.6-terra", "output_tokens": 44, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:17.994796+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "74c67b05-fd27-480a-9030-4b9e56d48d00", "result_sha256": "736baabbe2e9335edb6bbc3ae1d29bbcdd5c951d7dd33b41e4b2b8513fa39af1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005276, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.5926480330526829, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:59.061906+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b1654b5-c566-4244-b65c-a2f7ec25acfd", "result_sha256": "a9c125297c260cf0993a6ce2f787df038306575c1ae6b0e1e351a1cf0c907274", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005264, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.2805293669225648, "model_name": "gpt-5.6-terra", "output_tokens": 42, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:34.251884+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6455978c-f566-43ae-929f-6f7885423fde", "result_sha256": "a66866931291258c0f4b441f609699ccb4db94692729bb99e1fc1f30e3739d0e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005324, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.6420651939697564, "model_name": "gpt-5.6-terra", "output_tokens": 47, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:44.337367+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b35e04a-b7ff-4bac-896b-dd8c853861e5", "result_sha256": "284ff7e819285974c60885589274432b81e62f4843e89ae7943c42d5ab014e1b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0053, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.303852560929954, "model_name": "gpt-5.6-terra", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:14.744019+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b63aa8ab-ca13-430c-958b-31fab2b36658", "result_sha256": "e52f6fe461bf948cfd0383f92c02e84ee2441c178ed1159ef5d6adab5eae432e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.0020068070152774, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.356827+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8fa9a314-d0f4-417a-adb2-607ebef86c6a", "result_sha256": "e5199e6466cc4c3b6eebb2576ed3f7a8bebdc42eb82511664e6fcb944e3c72b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00527, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.6944514939095825, "model_name": "gpt-5.6-terra", "output_tokens": 41, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:34.277034+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c0d48d9a-45c7-4514-90a6-21d23b6343d2", "result_sha256": "d5edf89f44adcc0b25395bd859e84dd99de4fb028751ca26c5d67117795c04cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005474, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.605654745013453, "model_name": "gpt-5.6-terra", "output_tokens": 58, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:58.580345+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "69fb2541-9018-4822-a810-2b92fd21fbce", "result_sha256": "8d26ac772e252856118a9300b8971094d1c251f1c32661dabfebb6ef7d78202c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005234, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.4730375970248133, "model_name": "gpt-5.6-terra", "output_tokens": 38, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:34.526480+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9b3e3bc7-e784-4c66-b30b-5d775c6cbe61", "result_sha256": "428cfe4f595cb37b8d145ed9f240c2db25c3072eef2a7358a51c29da01a17ded", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004946, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.135414042044431, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:43.668450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b201723f-37c7-42bb-aebb-e687ff0fbf45", "result_sha256": "c20e1148eee82f001531accf1c5fb34d827d872b3405bc85a183979ec356b958", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00557, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.5069000920047984, "model_name": "gpt-5.6-terra", "output_tokens": 66, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:40.964398+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6819decc-5158-4113-94da-6e6d3eb6dd8d", "result_sha256": "ee226e732780af337e2d92c796afb20eb96f694c8ff4cb87fff949fdefa36661", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00521, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.182305386988446, "model_name": "gpt-5.6-terra", "output_tokens": 36, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:52.964130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6eb7e9f5-952c-495a-83f2-171b586b1b74", "result_sha256": "cb3c5611056a21682045b6d5a91c926bf913a969c9fa69dcb733ed964c8e071b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005438, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.6337794399587438, "model_name": "gpt-5.6-terra", "output_tokens": 55, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:35.279587+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e9868aa1-5121-45ac-8a71-7661dd1f940c", "result_sha256": "b311be1e6c4f62af221e5c3fd3e71546ea08bbb3cae49d98412eb9dd8b11f14e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00533, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.9649396570166573, "model_name": "gpt-5.6-terra", "output_tokens": 46, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:09.906762+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9fd9fba2-0d81-4556-920d-ff831904a376", "result_sha256": "64ea61a773fc6e9901a25c6401c35d60b865347502e0b1ff23244c22c8c42a15", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005294, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.480797695927322, "model_name": "gpt-5.6-terra", "output_tokens": 43, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:55.980374+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5197b3b9-72a2-40c1-8cd8-6b59e1f4665c", "result_sha256": "c1cb310f2782b759f6606e7b663451ddb922f519190bd0956680ecf37bf3fa7b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005426, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.618629898992367, "model_name": "gpt-5.6-terra", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:42.364757+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "746c99c2-bfd9-4f7a-b1fb-da4788957eec", "result_sha256": "93b001bae2fd8f2733816c104e50d37b593ca167c1e7ca06e3e5312775782392", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005354, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.289031014079228, "model_name": "gpt-5.6-terra", "output_tokens": 48, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:51.788179+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "fe235ce0-0a19-4d34-9054-07fbf4405c6a", "result_sha256": "162593f3adf7a22e95f4b25a2a8275adb51bab40b9c9fd4999e8562c49850aa1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.961956600076519, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:05.470955+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "422bbfdf-b1a5-41fb-a7d3-9e4f5d86a4a6", "result_sha256": "3e527fd27fd4b5c284d08fa1bfffb4f9a08a9a3eafae5583f4495ac801a3d8d6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0522027250844985, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:31.911430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "14cb8ad5-2a0b-4f66-95fb-71a51f3f0728", "result_sha256": "8f1a51b319c1acad5ca9669a06970055d985f5d6d4fb68d9ae7a0e20139d69c8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0153830719646066, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:49.868598+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "942b08b1-a2bf-40e2-8d6d-e5cdffc08c84", "result_sha256": "9e9ba3ceb89c17326775217eb46ffa75863f5d5614894a045e27dc03033653d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0805335739860311, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:10.067349+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3507ff1-f20b-410b-8163-a03c88ea125d", "result_sha256": "957bd57c65491aa81ae04eba7ede0b5d012202effeaab817f6584a2f1e3405cd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.0773984170518816, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:03.036996+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6f864808-6574-4193-b5f6-80e0c51efda5", "result_sha256": "c13132aa9da4dc073dae312d1a234fae73b819a34076e461eaf45b53f7af84c9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.3068594760261476, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:51.160625+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4189b1db-4dd5-4247-8010-796357378bc5", "result_sha256": "ffa295c547ffe461f0b03a4ad283de771d052625907a4e74e76d81373b97ee30", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.140832717064768, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:17.925537+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "af5ef259-991d-45c9-afac-5845f7654ce2", "result_sha256": "f7a21942369e7469d85f46dce4ca3cc4b58c478f050f123690bf4f79cbbc1e2a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.2545208160299808, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:06.208547+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "77307a4d-52a2-49c7-8f86-9bfcb49b9972", "result_sha256": "22f941585ddb7791e14ef1f6c12be6173c5062c33d4f2b081338b1d884069515", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 0.9973100199131295, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:48.744063+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e6467c46-952b-4bf2-9b7c-80ddc420b669", "result_sha256": "eb2713e59e7c85f8aa3aa666a4111f706f112a0b81b34d1a00d8a6c5a2fe9c82", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005034, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.115397223038599, "model_name": "gpt-5.6-terra", "output_tokens": 25, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:30.769521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "fba338e6-656c-4371-ad68-5b24c0557208", "result_sha256": "195274d06c76930211717cd76d4aea5b8bb9c2e32e1cd8162e6ff2c80cf21153", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.1539014999289066, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:54.998899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-5.6-terra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5cef2e15-daf4-4468-843c-f308eea8c9f1", "result_sha256": "d1771416fb2558569ecc8de43d56516b5f7365d49ea53c0ee5f37b02d2c9d1d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004902, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.1162295179674402, "model_name": "gpt-5.6-terra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:42.826442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "036888bc-bb48-4f5c-bfc2-959d32c91bfe", "result_sha256": "60eb38992bbc69b3156ec6b0bf18b486d340c22341a52666722f1911d979bb41", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.769518462009728, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:47.937087+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4ad21589-ed00-4a90-bc29-8db39018aea5", "result_sha256": "c82cd4d1172e19ba0ddc6ae3756132a3e1090983873c1f71098f0b295af263bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.0350079509662464, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:15.116047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "769c3752-f74f-4946-b0fb-725cd6dcb316", "result_sha256": "a0c20677d43e34a4e883d98da8a2b60c355a85d794fd35d936206eedc26264d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.6488714830484241, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:08.965927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b13f5d9-23b8-4026-9cbe-30c17f5d9989", "result_sha256": "6e21e3d4d8f1ee30fe620d976a8fe1a174ecf4846ac20e8164e473531a607185", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.7596855679294094, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:12.342678+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7622392-ddc8-48de-9c31-27900e10b540", "result_sha256": "eb141d8e68086795c2a7b13c3ad63bb51842e1eaf4aa3c537b488f82a88f5627", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 3.042258986039087, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:39.107428+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "885a2174-b0c0-4935-8157-8b80ebef7494", "result_sha256": "b821724cad728cd67bcd6cab6436c79142ebb26ab5eca9cfd890a51becb9e0a6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 3.360468010068871, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:55.060413+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e46510c-6655-4c72-afff-a489aa3d954d", "result_sha256": "04dd35cea4b6c548514a475902f8189fbdad6a934164943ccb5074515bf28737", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.565968323033303, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:38.625919+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "458c8ac8-04e6-480c-a95c-548fefdb7828", "result_sha256": "8c1c8784bfd1906603f28613dcaf326d5de18c6aaf82b1ffcdbd82f4f43db78b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.334380952990614, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:57.540862+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "274f6788-b18c-4f95-b486-7aee57f22b58", "result_sha256": "3dadb81aa3610135f589c79787944c9812bfb93b2fe4d6f4e530d0bd5b06d238", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 2.3245373900281265, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:29.843751+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8f3fd2a-ab57-43b7-8032-ff3c249e895c", "result_sha256": "3a5f9cc189f591a2d06df139951397d7fee581b147932bb2a4297d3cffd43689", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.88151810108684, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:52.206640+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "fb4c382d-cec2-4acd-9d82-38d07c775e48", "result_sha256": "555c31a7e12341cb527413a09a43e055f005fd294e04819859544b4df9b304cd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.6935948159079999, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:29.284205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "95efd5fd-c927-4a89-9d9a-43581b8ccd9e", "result_sha256": "2c72741c9cb5ed6f838c8ad660127cb5053e70fe0da453f5749e22dcb2fc86bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2396, "key_errors": null, "latency_sec": 1.8589964240090922, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:50.851327+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d1af9cfe-1f2a-49e9-9b22-5000fc7c69d7", "result_sha256": "bc6e463e15f4ec65ff1adac8908927367b14342198247104a98fbf4f0cbf0dc4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.0095601440407336, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:55.825574+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "099ae002-ff80-407a-a408-7b514cccc852", "result_sha256": "1754f2f782ce0f338cc057ef7ddc6e0d2ccac2d608c11d9a0a3c42b4d479521b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.7864581609610468, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:00.085607+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c5b2c30-342a-45ba-84b5-38b103f275d0", "result_sha256": "5015cbe73ef0b7748d88929ba29f3c296fefe95c2ed162a205d6af7317cfe963", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.655574228032492, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:59.799214+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e1d19a12-bf26-4f11-a5e2-59d6aca82146", "result_sha256": "d1a30939831ad2d9aa8bb8970504673c1493def91701a74d65cc67c1adc74867", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.8091006990289316, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:06.264854+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f2896d65-14f1-424c-8b8f-3e8c7425e62a", "result_sha256": "08d82739d0d7062a1aeac064652872bc0a6edc58d9edf1fecd9310c1f0b0659e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.9979225450661033, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:11.416933+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2bf51c75-3dca-48a6-8fb8-95345789b5f1", "result_sha256": "df172472c00a5a68f17e014771f4edd875d85b2a56221f042eb7e9bfc5fc3340", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.5823104990413412, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:22:10.839957+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b9bf8a0-2957-41fd-8bd6-d212f037475f", "result_sha256": "9f6919fa3e925d2f48de1b120d14a2b3bc4b4c04227e2e6634232879cf83a7c6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.681715608923696, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:41.944576+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f901ed13-11a2-45a3-84cf-69463f6fa708", "result_sha256": "770579ef46e3769b45f7a09e981559ff914f8851004975b57a69369e0766c649", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.3881495830137283, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:42.647178+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "51b42f9b-4b4d-432e-a673-53e9536952c8", "result_sha256": "16059d38c8aaecef63af5349ada85f5ba95e923347c6425edd128981e98a9d0d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.8008321789093316, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:25.612189+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea81a3bd-d121-4fd5-955f-5da8b14554ce", "result_sha256": "bdbd554fbd7bdce4f1d2899b1fe46e26be8294c023651db48855d9fe76bc6320", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.564132860978134, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:21.388297+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8ab6d635-f760-4c9e-9e53-02f4b9a50d67", "result_sha256": "4d1ac8f6416c3804e9438b375faa8c4166ef554bf754e2dec1550a68537c2993", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.5427031599683687, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:06.341794+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "151913db-d94f-40f7-9bf6-f13687481a0f", "result_sha256": "76b1d905fc06b732496ffa8c255e815e2fd0a671e002c371fa3e209228b7b6df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.6416736119426787, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:32.720520+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ad2555bb-07b9-420a-8faf-9ba1039b8714", "result_sha256": "fb1c783964c5b4b49177fe09bc43d9dd382fdf91c22bd6e53f2a4b8eb4e662ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.6397106590447947, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:50.114809+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9ebbf47b-84b8-4dca-8366-208296f06bc3", "result_sha256": "14e0ebe65a8dfe874653b368513d9dfadbc3379d5e0db3ec67968d47a4480d54", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.343282360001467, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:01.824081+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c88dead-6781-4c63-8e32-43c67c2e721e", "result_sha256": "4a740e4567257711cf41e57c11a9485b139de3919f1173dfb59a467d8d26c9ad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 1.7590133970370516, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:14:56.591269+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "835806a0-562b-41c2-ba30-c40e03b11ec1", "result_sha256": "c0e2c39c8ac8aaed9cdb0f246676cbc7950b7fd951cb76ce97e91d2c013e7e10", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02533, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2463, "key_errors": null, "latency_sec": 2.7476140210637823, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:17.529415+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c667cec2-e2d1-497e-aa96-b247d96c7bc7", "result_sha256": "d3d9af36ba71f1ebf369b3cf39bfc932d97b014e4a6ee78288afc457a04db762", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.6661378790158778, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:38.876966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "aec3a74a-8595-41f4-b842-fa3eb575b8a4", "result_sha256": "9e086863f1231123f547cf0ccdaf959300cec0c9add0ed0b1ffdd265e7513cd2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.1529681030660868, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:10.394942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "25fba717-d893-49d3-a2e3-93d1fcc7b7ba", "result_sha256": "6f7770465ee5593b0d9ee6d163776af3d89176b491fa917d0616f831daf95c08", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 1.6621509689139202, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:28.845216+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f16ada2a-a40c-43e8-9859-9a63ff0dc51d", "result_sha256": "32869a5f137a848ad3745f4c98c7baa573d3deba4c3be1350b5c635c22abd7cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.041030828957446, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:00.807888+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0e7d123a-8614-42cd-b6a9-993dfcfb6973", "result_sha256": "a13622f96e3f6c861de8a631d8a1e06ef384511d0f7d2bf4c8a381c8bcde71f4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.4805010419804603, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:33.028541+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "311b27e2-7486-487f-a8ec-bf17eb484ac4", "result_sha256": "b33f64313ff191f80bc95f695b03ca88765a8f53d2ed2dc5510490a0f3b0b93c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.1188070620410144, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:59.249284+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "dd1736bf-9580-40b1-8f75-4de210a7cf59", "result_sha256": "b3e09ca404361d65ade384b346c4581e86e2998fed7b422b1052f8bf3776ef54", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.6272938000038266, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:41.434186+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0fb6267c-8d79-4153-8425-ce8f699b708c", "result_sha256": "62c064115e8983e1fb8f95c0a988a2e1fe1277742be49b75186a34bce3bf9446", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02438, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2368, "key_errors": null, "latency_sec": 2.106917590019293, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:16.710784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c99da58-76ae-4b24-9bb9-564ce9a42e46", "result_sha256": "088a803726608ad136c652cd20463a5036785683b18a1845b9980517053b4453", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.7039277120493352, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:44.761865+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0ab5346d-6ec1-46c8-88c1-2ab2685bfbc5", "result_sha256": "3bce9b8d13351962e90222c7f239626a97a988d91b8d64cfa4cbac86ab980b8e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.086420360021293, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:31.773886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4e762f7e-75ad-4d93-99fa-e9de55610137", "result_sha256": "96979c8240aeeb0d087ccf990b878571a008801113ab20abb95ea382b8afa2e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.9518789410358295, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:55.031923+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "367803f7-7ca3-4a8a-a399-2145d31b1a0a", "result_sha256": "79554e2c93f51184d2dbcd7d6166459b702bb225fd2ac8c6d8acd52f33c58413", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02701, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.564992411993444, "model_name": "gpt-6-astra", "output_tokens": 47, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:13:55.680217+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9b93a273-9e37-40a4-be3f-6a1f94ccb895", "result_sha256": "81f589f25e908d9e876ace56bdc49668ec744539f5e3fbc39464fc3f6c5fe22c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.7539259750628844, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:14.626025+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c05a4c11-e7ee-4c22-8fd6-abb7bf3a91fd", "result_sha256": "221f893d23fe12495d7cabc5c09283b16bb7268981b6c8d8558e2eb8b056e094", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.5518138529732823, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:32.059917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3f2c1b4-b2f3-43eb-b73f-14ad5dea7d5c", "result_sha256": "e14439df0b63b87905a15e670da60c32a15ed42bdb196da3a5239ad555551b45", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.913281961926259, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:24.681929+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1d4bfad9-9829-4a66-ae48-3699f30ebc75", "result_sha256": "2951831a751a8bd21f8d1d4412eb2bc5c5d55e14de474eb492986454c69495bd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02641, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.127089389017783, "model_name": "gpt-6-astra", "output_tokens": 35, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:15.534349+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "dbe0c4fa-df67-4d82-b954-59494605b8e6", "result_sha256": "0f1dee8b11188e7faba6f61c97a76213d70640a5fa370ef2454f64cb5d7a0e05", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.738374840002507, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:22.757934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "78758829-15f3-4df3-bac2-a86b3d250460", "result_sha256": "5c71ee4cdd35641fad50ba35afc7a968a898c2e3390d81493f47e6e71b1221b9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.8806518040364608, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:21.410843+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b2fae74a-3aba-4828-8a36-0d99086e1298", "result_sha256": "a84bce186eee031c248aa007c4e35e5a5638fd1f7bce88a5c3a4916d5eaeeaac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.4300462240353227, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:49.849594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7d0aac84-7b5e-4443-b36e-75b149b57821", "result_sha256": "c238d81e00a25259036b74babe1084996eed2e508fbb9c26e432ec002dbeea28", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02606, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.012578728958033, "model_name": "gpt-6-astra", "output_tokens": 28, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:25:53.387249+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "18bafaf7-d6a3-4e0f-a41f-7b878a16db3d", "result_sha256": "fd3b0d76d16f1f11acd87f9e645f1465cec8a700b2563e7a7bb94db38ee27e91", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.5953790970379487, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:26:50.562674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f28a7217-5eb8-46c6-9d3e-fb8dfb83e468", "result_sha256": "fd918f302555520f91b5bb4a7fce4055dd83f472a985d816ec6ba871294d17ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.8258687130874023, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:43.343058+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a2b49e5b-81e4-492a-a9bf-8dbc2b7f5f5f", "result_sha256": "22ef0fd5749f838480fb76e4a8fe7d6ab9a531fd00ef483474f1db4af312dd88", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02536, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 1.6902852200437337, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:34.499766+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5c840ca5-7e34-4e9b-add4-1d2ec4a94fdd", "result_sha256": "4bb9329f2691f4142f54630b8a92a51e7515d39c7cc7a27c1eb442fa0f031fa3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02736, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.592380615998991, "model_name": "gpt-6-astra", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:22.602654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e6b79fc-72b9-4575-a331-f64016aa808f", "result_sha256": "480da69a117abe02a54bd5b4a99bb40d14bad19f75e3670f9de2eb2d1b5e3fed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02721, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.5115085219731554, "model_name": "gpt-6-astra", "output_tokens": 51, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:40.249448+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7278ef53-cff2-4aa6-8db9-7106d455643e", "result_sha256": "c24da2792b9b60f02807808103cb5b81ec6d3907a86541f697942ec3245b442d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02706, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.5068774240789935, "model_name": "gpt-6-astra", "output_tokens": 48, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:25.407911+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b909df4-dae6-4b22-9192-7804c0d92879", "result_sha256": "55a121b44429e548a4a486f39287c40b7d3fc552a9ef584bd761ee463855ddbd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02691, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 3.5810546409338713, "model_name": "gpt-6-astra", "output_tokens": 45, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:03.385520+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "64f05a52-b07b-4616-9700-40ade47a9abb", "result_sha256": "fa26f5ec4dfd7db6651e40c1447323d90376af01e33b2d01fa9958bd0e45ee65", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02636, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.132234551012516, "model_name": "gpt-6-astra", "output_tokens": 34, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:02.000958+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "326b8d32-d442-4e67-b1d1-50ad6d4ec883", "result_sha256": "1d504b2a5c067b0320ab21c3c187c5430251fe29995211f6fc0dfc867261a9da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02716, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.953108998015523, "model_name": "gpt-6-astra", "output_tokens": 50, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:56.254792+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e09621af-81e9-4c7c-96ea-93866fef7a97", "result_sha256": "b6b85c67a9534ba75a00913541d6638e58f760b3e3689a95835fd64f9e60d577", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02706, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.7393484000349417, "model_name": "gpt-6-astra", "output_tokens": 48, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:07.659815+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "98264197-d90a-4d74-a980-a939e5f28ad8", "result_sha256": "3b97d2c87f7f51069a09fedcf64f02e5aa77612043198ef0ccea0100bc4a8ff6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02726, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 2.668497856007889, "model_name": "gpt-6-astra", "output_tokens": 52, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:33.863140+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5560a026-a6c2-4140-ba7b-dd15fe692626", "result_sha256": "0d862b2ab4d01f480cbe79191560fddda141929bd6a3d6bca4c224ff0599698d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02721, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2466, "key_errors": null, "latency_sec": 6.417524794000201, "model_name": "gpt-6-astra", "output_tokens": 51, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:57.704470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "95d8fa49-047b-4cd2-8d86-10d393e1603e", "result_sha256": "3776b4b31ab802cd293b3ca3a08707dd717a11d5ae79588186c0e5612a236bd0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.6576178609393537, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:07.880862+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e3ed5d2e-777c-47c9-a786-a0430341590c", "result_sha256": "d3ea3a73d730cebb8136f68e517eb8977537cd3ea701d0fd86a742144c741d5f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 3.003540235920809, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:51.823319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d7a78ce-1dde-4d20-8f5d-6921ace1bfe4", "result_sha256": "93fd31c89c129d61ad0e0921152b545f365c4e67c5a2ca81055ab19da22e307a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.6872070520184934, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:26.301617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "251662de-137b-4af4-9066-f82616720224", "result_sha256": "2e038a890082c4c64509dd5c886978f7f7a40018b7580a99b7527c6d1ebe4f73", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.185379030997865, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:52.204351+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0eb511d5-a98c-4a57-be30-51cc04b6acf3", "result_sha256": "8d0ccd9f60ea2c51bee6de94af7fa37904ef8f0c0220e85080cb18c151be83cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.7741790609434247, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:58.672275+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf7061b4-30c2-4613-84b4-7c23676210ed", "result_sha256": "c79f12d12a45bb7f9b5b5f2e650d51b51ab4ec84cfec101c28095d3119751c2b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.821087777032517, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:59.929800+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "cc0698f9-4b18-4d9d-bf91-415391c151d6", "result_sha256": "3f31f5e36b0297a97f5b7bc9e9a3dbc9bcdecf77d62447d002b60263404fe9f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.159025999950245, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:18.194868+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "62826dc2-4826-40a4-8866-0eb5ed02f06b", "result_sha256": "cb7ab1f6de9033837cbdb66269e2a3bd74801f7fc4db9414431f005f35f9e094", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 1.8833877890137956, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:35.481521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "52de23f9-3823-4ae2-aee5-0c9d91bbc628", "result_sha256": "7645e422e16af17717a6c7fc3795e148215c5e10dba6b637aa55050162c56d08", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.1463881579693407, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:06.511069+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "520fa5b1-1720-4d03-81c7-301bf82e636a", "result_sha256": "3e563bb2ba16d861dd7a572464047fcf6c6f8f4012fc41821cab4e576e787433", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.0866045899456367, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:32.721974+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3a0ab60-062f-401a-9323-24baa61c3dbc", "result_sha256": "f3806576e486a5e1fc0b3cca7d4408168fe25ab0f8f7675f5fdf3941e1be7a9a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.0636834800243378, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:17.657442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b712298-d961-4478-8438-df43d87426b6", "result_sha256": "3a125df96abbc27f0f43e95518f9c215e37d79475afadd622c8def7ea0b6ed8c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02458, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2388, "key_errors": null, "latency_sec": 2.055056803044863, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:57.014901+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b8597b3-50f6-4e84-993a-1de74c5b0f60", "result_sha256": "7888cc8a5eed2f5d9ad527a8fa3fa609b8858db05e71c61be558271ea406ce41", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.2853322960436344, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:09.580526+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c920b800-3b36-442f-addd-d907636fe2d6", "result_sha256": "56075ab85834abd4ef00bf1da0e1de0e4c3c189b6c444c7f99874b83ddcb7188", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.0112266129581258, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:23.390608+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3986a16a-0f59-44ee-af80-ce9f446d76a0", "result_sha256": "49be4e79128ce66bf3a0aec6105260ba0a86547bf2250c2c56d9428c4376d2b8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.8677607310237363, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:09.875334+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b0ea7a73-aa07-41d5-b31c-2667323a2371", "result_sha256": "0de56fe53b3c288c1a3bab72d2715e8ac1283dddd8a7977016711cd03978631b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.5179676009574905, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:41.273024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7523004e-890a-4977-a1f4-a530ed31b5c6", "result_sha256": "5ada595cd773df96bc15dd8019e5945ae0b0cb4b54ee054849a93ec1b5fd4ac4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.4856990430271253, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:47.704197+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "721a4ca7-5b8b-47b3-9a68-f32afdab065c", "result_sha256": "5e81086d528e803822add3d3fb63237e476817e0141641db7acb94fc13186780", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.5876195400487632, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:14.516861+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e67e5194-bf04-4e58-a0f6-df06fd7bd61a", "result_sha256": "9008b20c57a3730ddb218d87196ab96a4e72cbf8cb5a4c4c0db0aa54056b625e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.0822818420128897, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:55.088761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "160d9c92-51de-4b26-a1d8-f6715570da36", "result_sha256": "87b7241e1d628f32e72ae2fb2bc1b85ad3b2c357bf7d1047725a156373e4c843", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.106411244953051, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:58.890384+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "23b693a4-413e-4e1a-895c-ad3c7700ba57", "result_sha256": "946ad4e1a5a37aafbd0e44b97f726bab9d25755482958f8e6c724a37d51334e2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.9692281269235536, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:37.571633+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "33689e9e-50c5-4746-add4-0ee9832fb9af", "result_sha256": "a52429e95bc1d23a2e50a0877a5fcaa6c399dc733fe48f769e73fbc0b11ae4fb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7086975959828123, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:28.808088+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3cbd17ae-d9e5-4262-8733-48aab77c3773", "result_sha256": "20c67a41706732395abbdb2cbd979c49550ed7ceb80a7fd02fd686e28cdb782f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.0042898550163954, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:32.967708+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d8efc98-abe2-4914-8bf7-6c465cffbcaf", "result_sha256": "fad16eb08099a0d3bd67a8f7fc28756a170d450fc577c904a65cf36631740e31", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.8263020509621128, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:20.702188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "db9e6c60-a5e5-4a05-a02c-c33dfbe0b716", "result_sha256": "f177881920720f0915ec4361ee2ee5add899d611d4540b5c20ca63493469befa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.8923140919068828, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:03.318304+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7c919ae7-b6c4-4878-bad3-cc0593b62b7a", "result_sha256": "dea85758d2b333290a4c717cee74160cfe2e3a3123dfef9dd7a143e8ec604484", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7147251169662923, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:29.298405+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b7eda13d-8478-4974-b782-f0f35b07667b", "result_sha256": "a0181eefd0aeb43f92e01aecf9e1f4ccf68eb12f94d3f74004f76da20c799ed8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7819066270021722, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:11.467737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8b00dcf-92de-4e59-b18e-3ab747fbd9d5", "result_sha256": "144076119b3985722601ada58b57df0ac7ad9feb816c95f8f28b7950fc665c78", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.3566450669895858, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:27.923126+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0bf3d72a-c237-4d75-9e1a-06af0e1b6689", "result_sha256": "024c7cc8270b3d28f7d993d202497fb5f4382281f4831ea3a9ce09ff2469671f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.7022918849252164, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:28.026123+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c0ae9f6-36ec-465e-a9e6-01edb805df91", "result_sha256": "4c869be01999d24ceb4ca816ff9adc09cfdb59c4bbd97d59166b1661a383d791", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.6678674010327086, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:38.060635+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "70236a30-4ee9-41fe-8049-f800796a959f", "result_sha256": "bc0859268ac3dd2dbefb1e96bc22b4ad0ef2b382ccf59bdc48a1ddbe97856aa3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.720175395021215, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:57.016512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "42f3b987-a662-40e1-86ca-79201ce78f28", "result_sha256": "0b34818c452e022eac118222066ffe86660959cd9ae1eb42a783b00b37ae9826", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02454, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 1.9510197109775618, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:24.480808+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff451f7c-1f8f-4240-8004-62917703c933", "result_sha256": "7f0415963d943092a388ba9503293cc76186eb69f64e6bd03582e353f47a9d26", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02669, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.1573236900148913, "model_name": "gpt-6-astra", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:39.278572+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "87136214-ac81-4323-890a-6e54bb2351a9", "result_sha256": "35cc8d0cd948d4d2398299f9727249b6087916409be487c63376dd8d79188d35", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02654, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 3.0680234589381143, "model_name": "gpt-6-astra", "output_tokens": 54, "prediction": { "choice": "A" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:15.626402+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c406aca9-6e43-46ca-9b20-67801a19f83d", "result_sha256": "f2eb39113f95e41425878faa6710da5da824e345f91e5a74daf769a71f9fcd12", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02679, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.94752228888683, "model_name": "gpt-6-astra", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:08.220508+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a6191436-978b-4cb3-9d2d-4d45b561be83", "result_sha256": "bb4c4e1ea38b3c1f0f3718a0b21e112f317b07f75b6fbdde24f380c8a198bf99", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02684, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2384, "key_errors": null, "latency_sec": 2.6291073210304603, "model_name": "gpt-6-astra", "output_tokens": 60, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:19.355897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2904357f-d382-44e9-9fa0-714954faca36", "result_sha256": "21d7249930c6e138e554e47fe1f6a3a62c6842f6e52340f9400172ccd0b91593", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.0227839429862797, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:04.609199+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5a768bec-0bff-41fa-9dcc-d6cb122d0a29", "result_sha256": "f311f29d66f0c2a0b5b57c68959aeabbcb1a4e4946086e31d097e3e5c5cc74f8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7067234340356663, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:11.105581+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f932b7e-5164-4251-beae-5e54cf233792", "result_sha256": "9a9bde3fd558b3270111b63695ea03bc4bc2050e0d09f2caa45fb66b554b72f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 2.0638372290413827, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:02.083894+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8750baa6-202e-4ecf-9668-906b272ae6f5", "result_sha256": "3e2319ab4c00818994106652ec1b5f7014fdfb45903845638be06aadd98d6529", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.8165542220231146, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:51.973964+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "de065392-6257-4fab-8892-0b0463b3be88", "result_sha256": "8c81e45c4a96fdccf85edf81caaa9922bf88f879e36fe32c655531c254ef6350", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7256433219881728, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:31.146435+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "aff94c06-b757-4de3-bf4e-3b87ae37edac", "result_sha256": "d2f7da890811aa6007bed49905f27189770a280ff3ed16a3d88a8cbe6ed63ec8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.264407465001568, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:45.058868+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5d0b0fd-3a95-4b54-b5ab-b2894d59cc73", "result_sha256": "8d78dc39a5a28f4bac92f5797dee721764b6118bd160f355992128dc8f152110", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.5507899520453066, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:52.182679+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "843bf84f-1cd1-4f37-8321-db0fdbeb39d1", "result_sha256": "74d7d769ff3812670ee207ba85046e3a4c9e53854f7c762a05c208f31e916d43", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.8442152760690078, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:03.710502+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b02d19d7-6903-4c52-af13-c2199c91f9fd", "result_sha256": "cd1e7c6bc5a235407bfb39cf027c12b9f4eacdef0713c6936f323000659286b1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.4975779470987618, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:12.365195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea00a540-9945-46ae-a67f-3a55fb0df873", "result_sha256": "4dfa0a5677c0f778b4b121f6e9ad370c1f2a39f9ccb7059c36118bf4a62ddcbb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.7065674240002409, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:26.004279+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9ad60d0a-6eaf-4486-863e-58d8d97af445", "result_sha256": "a67d4453b0a8e0f49ca9879a5724c87d739cbacc503889d0a84e6c3ce4daa99d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 3.5027740070363507, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:53.818191+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "24f0b53a-e96d-4d5e-80a4-216c0eb931ef", "result_sha256": "bdfcec5f7ced926c79f57c88fb437fffaf413a6fb1949ac8ac1182cad3e4aa1d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0244, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2370, "key_errors": null, "latency_sec": 1.9620658629573882, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:50.960324+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9a1cde7a-165d-483e-b6d6-f6a85b2fe8ab", "result_sha256": "1befa437e098dacdc1bbc7cd36e5484b6ad3b43293b5ffec39b2e5a9c3c86706", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.6752977590076625, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:54.277334+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "44ca173b-1ad2-4b9a-805e-2c3f00edc1b6", "result_sha256": "1598becf6ab10ec088d45f64774c9392b722f890e73354a74fb90e78d8c352ff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.7188507179962471, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:08.454057+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "822f0fbd-3d40-4171-8680-2bbc36be2347", "result_sha256": "7e48129a03969837c289bbaab65e979e8cbc09d82f53a7b84d836eac4555f48e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 3.419038404012099, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:44.377399+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "91157dfe-ead9-4309-81d2-b3a27b6766b1", "result_sha256": "d029114b1e8f3dd446a1301d1e8f14010ccabd3a06a273d088462273584cf982", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.7542114569805562, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:03.327282+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "95dc92ff-0400-4b74-8433-f123c21165e6", "result_sha256": "99c2882a7942ef164f9fecbf533510e124dc3b3bc738d317c20c9a2d0f549a3c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.074014561017975, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:25.294188+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "65592adf-2526-4c41-a30f-ed3473c83085", "result_sha256": "34937d1c8657382f97f76fadbbdc66f575933c3cfb56d09855172aef40f38803", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.9845949719892815, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:19.303682+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b694d672-6d4c-48d4-97cc-a73c0d1e7bae", "result_sha256": "3af12b2cede0764baa1a66ff293ec2994d0a614a91a9b8f429ebd31eb612ab49", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.7905632550828159, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:34.670690+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c6b9b12-9b29-473c-a668-c8d7597b7bc8", "result_sha256": "8f986ca69df738db99b0d02ea941e524a3ca5bffe28497b57dd6f95d6c44d336", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.5807208640035242, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:39.314000+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a56a6701-cace-46ed-8334-f81f922a2e75", "result_sha256": "7b7f31811a07226750c123e5b26a97ed24435a9da23e4a0f0d6b6214c0615e85", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.363896321039647, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:57.195915+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a02d0941-9ab8-4f62-8a4e-f0f8a85de5cd", "result_sha256": "b601c83bb2b30a802b392e3656d7044a3a362420c02c565aa6acf7db63f27a90", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.6109942119801417, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:04.526082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b40b3cd-7bbb-4d97-afc6-a63c962e735b", "result_sha256": "a82bae2047cb35944ce7ecfebfbf3f5ba4fada808ab62d3dee20eddea0a7108f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 2.1921137870522216, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:26.808166+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2417717e-f480-474c-b973-cd716804c3a4", "result_sha256": "81f5db07f64d1ce9ee66895a02db31abfc5b49f378ce205ec015f722bbb04b22", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02463, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2393, "key_errors": null, "latency_sec": 1.6470684799132869, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:16.076801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "18c6baf2-b8b9-41f4-b7d9-a5d9837b8a17", "result_sha256": "feeb698b8244acce4e6d080d59826df8e9b6d58be7c3bcab143d2517c8ac3766", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6768909769598395, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:42.733620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc42db45-f558-4c61-aedd-e14ab61697c3", "result_sha256": "35e4c0c4e7b0db9f8d7414b4d6002b6703665109ecf6192b5fd30cce4fa3fe97", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.8571833278983831, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:14.330166+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "148e8a5a-1b4a-4ae3-9653-5ef13347097d", "result_sha256": "e8abec7bf811fb05804b09492e59c0b492129bf7760ef92e4875232ee229745d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02635, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.175622513052076, "model_name": "gpt-6-astra", "output_tokens": 49, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:28.285905+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "574f98cf-fa37-4680-8d0a-80b8ad439784", "result_sha256": "29ee707f32e220e586a1f7e834d87bf54562c835bf0d3d6600a516e3972e1d82", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02635, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.4099687569541857, "model_name": "gpt-6-astra", "output_tokens": 49, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:11.338776+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5314eaec-65ab-4677-9bab-93820f4c852e", "result_sha256": "5444607cafbe706285f5178288182179b168e331cfe5bde0d9f8329f2e0df303", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.4050529019441456, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:06.985144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "db6d9123-4891-4649-a884-8009cc87913b", "result_sha256": "d65bc5898635cc53308997a12b5da5ddf8d59f528bb0f31635574e6979781b20", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.214155350928195, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:19.858514+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a63e58c0-5e9f-40bd-a5b5-a042ed6e2bf0", "result_sha256": "6857b1afa9ee512ce46fdc1587bf485795fe107b955668816a2205499e4fb627", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.886167115997523, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:42.706041+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3367162e-741b-4627-8906-bb92560f4baa", "result_sha256": "bb90b4fa4142c3f6bacb5334fd95a2e420927372adfd98375c3ff4728f14b089", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6564099489478394, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:29.782148+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7a9ebc3-c9d2-4fb0-a48f-4ad5475cc59d", "result_sha256": "1d21b3ff0b32ec1d4c81130a5cc7a951f10f6187cf8176513b36d4b12f38e518", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.2042617690749466, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:46.594503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa445c56-172b-4653-9e13-2d4c6030476f", "result_sha256": "aedd25a78e04165bd7bd9423d71c86b2ac38746a50c2ea3631f32ca87b1825f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.8799487149808556, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:22.561208+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3066c993-d4f3-43ff-9ffb-8ec1c86f24df", "result_sha256": "35e7fe6e760cafb620f4c7666108343b800e2f35e25e7fb8806c60791e16edc8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.8068501899251714, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:22.891554+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "de2d5280-ce0e-4765-a57d-92026d4f03b5", "result_sha256": "6426652c3cf2d8fc0e52555837400fc592488b8a899c140f585d550c790788f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6713304129661992, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:42.950914+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2109e751-4e1e-47e3-8e7c-a53d27623fd2", "result_sha256": "1473b29c97b96d7a938fd794edc6f04d1cfd1ae8fd2aac5bdff797631c04122f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.9234145240625367, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:23.092360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6f7016a9-87ce-4bc6-bea4-c8ce18150239", "result_sha256": "52f76e5814a8f649b0ffd0554589039bcebd5e96510d32c5ac8616f96ef602fe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.6496245269663632, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:04.371588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e1b64e2f-be13-47a4-bb4b-c7270c0dfd1b", "result_sha256": "3f0aff3788b9cc7d08b8845c73cab1bea8c8e4e4496c3aac41942557577f8621", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.7770795350661501, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:21.806703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "dd809f10-a864-4171-b0fb-c219d375d752", "result_sha256": "6e9c43c0884138fdf8d8bde6ce1d08494a3cb3f653035ddebb4cd529ae9600dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.0697989619802684, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:38.934657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "43175770-759a-45dd-8749-c6c0affafcf4", "result_sha256": "fc5f13512efd55f70dd42d7ae6d11ee9eb55c56d481a81f4370a9d3caf347bec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02589, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.26736388693098, "model_name": "gpt-6-astra", "output_tokens": 37, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:46.577648+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "deac0f84-f1be-4031-83ad-89f6345a0d65", "result_sha256": "e23099809dad09b5dc6458df9e3593ae32ee394314e585ffa09bfdf95df17f2a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02609, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.538791846949607, "model_name": "gpt-6-astra", "output_tokens": 41, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:09.564738+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b0b7aea-30a8-4b6b-9a83-6fcf16b4c9c2", "result_sha256": "26c4215bf5cb1aee304ce277b0fbdf720b3ce9ca590c5f8e1abac1e1241b5b35", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02674, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.7181236629839987, "model_name": "gpt-6-astra", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:43.568841+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf5fdb0d-1f05-4682-a95c-bdf659b663f7", "result_sha256": "d33caa54c432b70f6e41f8d98318a244cd354e6ec8808c5a9645c8600886424a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02669, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 3.5392844130983576, "model_name": "gpt-6-astra", "output_tokens": 53, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:39.411430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "92d6e9c8-43f1-464c-89fa-8897f6a4d0a0", "result_sha256": "a757b25507181bf6219229c9524b38f9f940ea64f760fa4bb56e9f254a4b1905", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02639, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.8057968760840595, "model_name": "gpt-6-astra", "output_tokens": 47, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:00.889950+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2f345472-4eda-4024-aac8-5e35daaa5d89", "result_sha256": "b1e2308dd5f063a707d85ca939bbe92f8c0f908aa4b03e92f2d2384f08f37c23", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 1.8075546741019934, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:12.169753+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "650a65c8-eebe-4ef1-87d6-fc9fd528c93b", "result_sha256": "6aab532b1fa71532527fa7f2cf4368cf432968cf71a586dd3e69ef6ded3fb313", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02474, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.122300901100971, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:17.799435+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e6ed0a65-294e-45dd-b414-3e2ee8af52e2", "result_sha256": "42f3bd2f989748242f1693f0795dbb7646cc13eac957dacdc86591d4d40a077e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02639, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2404, "key_errors": null, "latency_sec": 2.207453773007728, "model_name": "gpt-6-astra", "output_tokens": 47, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:34.510614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2bfe2dd8-6724-46fd-8917-67c0759d606c", "result_sha256": "2084cbd144bb1591ed73eedca7e09cf1b5ad556fe4e85684fe432079bb0b8baf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.605053577106446, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:54.096154+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "40c6bf23-0254-4a0a-a31e-62b36315a97b", "result_sha256": "5f4378955f6b47f409fe70d7b9eed5d3b6a0c0276c2b263aecc4778e76edc84c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.566450126003474, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:59.411226+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ccab72df-4a7d-4274-b9c1-41e6938c5e36", "result_sha256": "b6e8dcf70a482e7b835b1af8f26fac523e0c203c36444ca1db6b2c8bb29763bb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.3254287829622626, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:22.175709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "528117c5-61f7-49fe-8baa-6dc7ddfa29c4", "result_sha256": "baaf6f3773b2fbdc1fb00500714a788f43e85ad98557a19553f7a0acf5fbe554", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.9788258839398623, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:54.539674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0533b225-b57b-4bdc-806e-4f1263f318cc", "result_sha256": "708e039864e9038cc4a4033bce4c260d89199a7199c7ce1ea03baf3909f353d5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.954364636912942, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:46.403620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a9a1406-60a3-4023-9f0d-abc73d0139cc", "result_sha256": "989d25bb0590a7f7612b28a343eace7c4a78371ee46762010af57473c1f459dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7951852089026943, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:20.506646+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b7cde397-a737-4992-9a2b-5228b847cafa", "result_sha256": "eec38354b86d4b4ae16fe6b9ca8c44f3a2b2c542e6885348a39d52f0d7c96107", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7888805479742587, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:27.084877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ba95290d-9fae-44d2-8ddf-e6c140986298", "result_sha256": "cdcf1774a30d668d992d1cec99c14d8e4f58c6488aeb19b382b383ef473731cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7059440129669383, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:45.611693+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e6e0f884-4414-494c-a59c-619688f379b5", "result_sha256": "ac93208336f65fb6692f980c093e86a90e9fe368f673d72b614ec89992b44246", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.1376475270371884, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:09.659767+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "61258106-0781-4f41-b4b3-a0840ba000c6", "result_sha256": "09866316cc28bb3ce4424aae7e458464abbe18b6db85eee5fcf8d4dc616a0584", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.0246825329959393, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:12.956236+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "07e0eb15-2d10-463b-b086-4795b35a5c5b", "result_sha256": "48ed2c7a279ad419ab8256069885ef1c0a78d5f0c5b28f615b7458a6d1949790", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.8976690049748868, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:33.012010+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b0fd7471-8b1b-4f8f-8dfb-584619c57e80", "result_sha256": "6437c28903e0b84dbc8f7d76cd82d991cdf46f81002d0492e365a66567cf740e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.693875737953931, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:12.240287+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "889bf78d-897d-4133-8cd9-73aba2ea80b3", "result_sha256": "b64897877b97b82d8f6fc2eaa196f531b6f55741626ef8d15303d365949d4918", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.607061784947291, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:45.551082+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3094b63d-c431-4cb6-83ba-6a544a7e8647", "result_sha256": "a17ee347ffda70e6f8f132e278d86eddb7752b902d13010c5ed1935cb99a42db", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.2433836930431426, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:33.986786+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3a2771f3-ea51-4ed1-a492-a647399f89f4", "result_sha256": "b58f551a19eef37060a43108908bb5e35e3684c47549e5ff34c8e750629c8f6a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.5505200320621952, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:37.539649+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6e062996-b8da-4f05-8637-b050a5d4201e", "result_sha256": "6046e8695c134807c6ba161244d073538a442b93a24e45c09ec77b5787549ba5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.164008477004245, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:33.406847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5a2e679d-5e31-457b-926f-797ed9d9fd3b", "result_sha256": "d6ac1616954a77b26d7c8b02a55c7f847dcc0d06f4bc3ebe368e335ab83d672f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.4470467569772154, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:13.012629+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "fe25cb9c-8a0c-4ce5-beab-30ea5b927e34", "result_sha256": "0b2018a9288c4e77e479280fd5cd636c0cd55875dfa5219915e39b4fb1dc7578", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.9578714109957218, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:15.364412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7e9e9731-e533-4996-b84c-151898a66350", "result_sha256": "15672c47f320d19e0d231a3b35b11372bb23cae948e584672a905573ee4a9109", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6589966209139675, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:53.354077+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "89a31d0e-f5c8-4eea-a288-4991da80c254", "result_sha256": "d4002e023e0d2ae4259bbf97aea92ad3726cefb5da7629a888817383d61b8da6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.7133158709621057, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:46.107625+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2f3dd49-1d28-46ea-94fd-340fa3111157", "result_sha256": "342bb61cc7f3a65022b5552ac0266407ecb03be4efa167f4f81829320c9a5ee6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.6969293979927897, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:28.930863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "810a1b26-03c2-4cdb-b8d7-fd2803f32ded", "result_sha256": "3efd7146b7c87ea5127b5b57ebb0d9404eab278c1518dd740115c44f0281ee74", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.2287813019938767, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:05.698308+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4abda47a-d0d9-41d2-80d6-188aa7947f2b", "result_sha256": "5201c7d910f3e44e692b010680cc70c9fad58fe182b51aa0dd6037549cf34db8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 2.353095152066089, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:38.302174+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "70b3bc7d-5b9b-4abe-a5fd-025e7c07440a", "result_sha256": "826e7986bb400c360d856d6cc0d9e8a4c744c095bfe154897f2f51ff1141a538", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0246, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2390, "key_errors": null, "latency_sec": 1.962256320985034, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:06.240990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6456f4b8-7f2b-42f2-b641-f33158a823f5", "result_sha256": "ac1b1e347ef027f3868d791cfd6595878e64f2db04dae37c697ae5d0c7226672", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9335888529894873, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:21.943614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "18663a5c-5f04-4b7d-906b-a061ed7529b6", "result_sha256": "88aaa4ef25fc5910eff391e8597e21cc536d9bf8078b66a22fef79b7829f4be7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7259342690231279, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:29.509695+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c9d98c8b-1ae0-44b8-9a95-783301765163", "result_sha256": "e9bf8f37866f029ef5b6dd946b5f77beb6b1864fb953397060a1bc86db7b87da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6619773249840364, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:28.551768+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9ecba59a-babf-4176-a027-1e76edf0784d", "result_sha256": "492e35c6dfca26ef40d97fea87e11c61337feb2b5788facc8bdeea3f46fa5e52", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9064801159547642, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:20.513442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a47d92ef-efc4-4358-adba-ce49d7da441c", "result_sha256": "f290e87b2fd1ba3c4de0b516f9094ea73a0bcd87e9029e264a4f075d548b9ace", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.0162298530340195, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:03.537263+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "8a807286-40a1-4f38-bbb6-499110f1add1", "result_sha256": "e48ec20530bb4c6fee62518855384cfa456dd41298e28c6d21da3ca8cc628795", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.067502925056033, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:04.046926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5cfb82c8-dc5e-41bc-bdca-7bb40cace0b3", "result_sha256": "9492810df61ea40fb7297946ffd18f451f01f037e501d33b632ff9569fe50c52", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.6506151809589937, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:35.248867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5315663-fc8d-4392-ba1f-1a278f2e5768", "result_sha256": "f976939ccf53e935203df18195c5fa28e03b51b9ed1c93e765eda3b181e2de5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.407146377954632, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:25.507730+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3cc2be84-dedb-4110-9816-66569329fe46", "result_sha256": "fdadbb7944429e8bd86528de31138b0676a39bd4b533bf173db4a576fc378cc3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7405636650510132, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:02.952226+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c8da306e-f6f4-4542-a5a9-072eeb512554", "result_sha256": "2f6e22eaf1e308a69b5026fa9193abdf92afc38cabdd7080d539f2e6b35ffdca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.421294468920678, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:28.987926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f526f5cd-199f-45b4-8168-b824d098eb57", "result_sha256": "c6ca2f2661ab16b7db321ae484567d65eee814c9cc7feed27706d803214da63f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 2.67257488402538, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:17.415616+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "905162a3-9843-4386-a06c-6a555a7bf1ac", "result_sha256": "2aaf582c7de22a632ee43aabb3d80fbd37c69cce2641c30f5d6e4c14ec5f53c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6498645480023697, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:44.005997+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e19c9eec-9fb1-4de5-9ef9-981f49c82092", "result_sha256": "139d5e57d0c65371312fa736c577a737050054d99695ea2002f961a3db8f4cdb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6773004749557003, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:17.540895+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5a3b33a7-02b4-4051-adbe-804999907aa0", "result_sha256": "7e10d565c290d0c6b5d48bc5324da47ccb5d5878523111dd170d6f9c308b9232", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 3.7697309349896386, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:21.065517+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "90f9c8c1-1096-4083-8064-6077638d89c3", "result_sha256": "b1e15dcdd446a1de773f6b18397a65e6e822600ea77f74c9ae57f8e8b5daaad8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.5733935890020803, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:35.607358+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "cafe4e99-1d6e-4c9d-acba-b7b09ad8c056", "result_sha256": "62cbdd4903f7ef9dc98170073ff81499783b5c225d768f9a9465cc681d1e021d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.8264181669801474, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:06.263073+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "fdf43fd2-00c3-4abd-8d0c-753b387584a4", "result_sha256": "2f39d068c926e2ff8ab61eadac2e119d3e2dcbfe067c1caca458f1824f00f35a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6572157909395173, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:22.382996+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b3a5ae8-08a3-4097-bbfc-bedb4748864b", "result_sha256": "82b52c37728c638c21d1acfc1983ef7c67dc2e18eb101f0a2eec3059a09fd4d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6502706829924136, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:30.869548+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a55f2a8-a6e4-4ed8-8ffc-b60132463527", "result_sha256": "d9aee48568183f40f81ae123e9d16452e05b8ea93af8c9f4c1dfbdda2573de0b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.65635155106429, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:24.105254+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0f5f7880-955d-4f95-adb9-caa417a58236", "result_sha256": "78a7e9da655ea3fe71a7f0b5a90d757ef892fd950afbf4ceede31c9ed559f835", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.6129046909045428, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:11.646637+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "85588c43-c30e-42df-a5f0-08f9971b7e47", "result_sha256": "edfbf517963f19e588dc1b73d5bac2079c5df774817ca49c4558d9de4ba1ed49", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.8533059089677408, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:46.629851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2250c7d1-968b-4375-b478-4e72fc2f08e8", "result_sha256": "075019c61bfd7bce7fa8ff8bc6be60f6bc200044f7493b2abd025c36c255ae26", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.8374732219381258, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:58.130606+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "fdff8d66-0114-4f56-a7f0-8df04f55eb9b", "result_sha256": "ada6dac5eafc2399d1d52d008cb1d22a5dbafbfc4d8d442c3cf2b42503ce2d2f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.7253484439570457, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:27.016201+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "01386172-d8da-4758-b792-648dd2e1263a", "result_sha256": "b3662ddaae42575d60243df295400e1c962637d4958fed10ef98a7400b379aac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0247, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2400, "key_errors": null, "latency_sec": 1.9697817480191588, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:48.813841+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "b935a783-26c9-4cab-8202-1f1f8b03c012", "result_sha256": "c0aac45cebb0f7d81ded9694971af37b5eb1f83be286dcf26e7f3cfb2b483fc5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.664481128915213, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:56.490842+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "732b31d5-1c3f-4d22-a586-8ca71a927862", "result_sha256": "43eed35a9814bdd7acfe1b68ae7f6a9458aad105acfab220f238799ca00ae509", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.9721842280123383, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:18.699994+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "de15e280-c6fd-47ff-94dc-9a6295edb007", "result_sha256": "62cbb4fddd598cc7d65441308d746d022af0b1f7c51a9353b80d8c5d9ba47e99", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.021994363050908, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:11.241941+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ee8b3b9-7c99-4d5c-9553-16f561dd7985", "result_sha256": "f456065f7228293e06223ee11e0fc2c4409ad98d6d7cb7d470aadeec200bf8d8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02609, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.743736026925035, "model_name": "gpt-6-astra", "output_tokens": 39, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:47.135667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ee5b7b04-32fd-4c1b-b077-ef95e8ab90f6", "result_sha256": "1168dfc11b6f7186430a717285d620d4b9ca76aa5632679e0bca4919f66c68eb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 1.6342774219810963, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:26.596583+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7331b161-147e-41f7-93ea-10362db8e7af", "result_sha256": "9ef27cd2aeb514e58407c5717fd7106ccdba8a12c3fa7f792c65885576349af9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 5.212774676969275, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:35.019911+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "2b63b413-db24-45eb-8759-ec388cc13823", "result_sha256": "3378d9ccbefe7ffa6949eb76ca9e5e5283fa84c694f131eba369d4a44eba264b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 3.199804740957916, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:14.627732+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "11641791-ae31-4080-892e-a80c73d9f383", "result_sha256": "bdfe704d595f81e021beabb97108a649e664aaef393f46b20979b0a23f62a3a9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02484, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.6181842000223696, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:44.367770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "dee3ba87-9712-42a4-b9ea-7994c5b8108c", "result_sha256": "bca37f581b74005a6f8859a16fbdb0d4a0ce0b1104f4be5a3d8c6f221f404d5b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02629, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 4.243764049955644, "model_name": "gpt-6-astra", "output_tokens": 43, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:40.201712+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9f28aa33-c0fa-4814-930c-0a12957d1af7", "result_sha256": "4fb352b7de643bf2b6991e278323639df51218c68ba6482527bf5b54e6ae8677", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02624, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.7700190019095317, "model_name": "gpt-6-astra", "output_tokens": 42, "prediction": { "choice": "A" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:47.719876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4595fb24-dcc6-4e5f-9928-b65229b79f2a", "result_sha256": "707154d503f863eb12069c56efed23a203d0ad24f6dcc072073dcf4b076307c0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02639, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 3.769161332049407, "model_name": "gpt-6-astra", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:34.169117+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d64adb11-9b1d-4d9b-849e-47700644dcc2", "result_sha256": "109f2d45e0c4720ce61380cede5a02678bfca4b97487b80a679769ee924f60e2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02634, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2414, "key_errors": null, "latency_sec": 2.434617858962156, "model_name": "gpt-6-astra", "output_tokens": 44, "prediction": { "choice": "C" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:00.396879+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "88b25111-029f-4467-81b4-b3090826576e", "result_sha256": "f174d391e138aa6f1b156867709bd96c232d828314c3e3c9b242cb932fae988c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.6074183869641274, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:25:50.228926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e01f8c02-631e-4496-a60b-b5bde04bc382", "result_sha256": "6f8201917d999818453f42a683fcf953e18ee871c005e2067cf76d12862aa585", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.7235110480105504, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:46.983274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "af04311d-8e44-4cae-b4ac-664a4fb1d974", "result_sha256": "61a6c71b2739cc4cd4449e55bb71430ee938e41132e4e589c73dc064dcb754ad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.6049665519967675, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:47.628989+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d966a7ac-af89-469a-bfd9-4cb163f9eb42", "result_sha256": "cc8f35a99cb65a05a6dea77e934adcab361fe651ab6fdb9248e43d4de8c21f36", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02718, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.1983023890061304, "model_name": "gpt-6-astra", "output_tokens": 60, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:15.703504+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a482c052-244e-4099-b17b-f13d8ead5578", "result_sha256": "80cb3860457c8b46eb7e9afd5707be558d2c2252b05ba53d4075a66c3d4c973f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.7790880149696022, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:50.285899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "38cb4b11-e7fe-4b57-9e4b-a45988eb4d85", "result_sha256": "67db8673bf00438d409b4475f23f362ba72fe29d20df744c297edee4c0bf3559", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.6783430599607527, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:38.052596+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a2f9a90f-9290-4c2f-90ff-809f2c22f07e", "result_sha256": "e6786213f3329693b06b59c12dfb8de6514fcea2808ae9fa97a84c368377eaf2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.9119623580481857, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:13.467648+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0ca6d84-92fe-4e2b-9abb-0d340e8ed0b4", "result_sha256": "9c30b45253a01eeb93808af8e5dffe90d23c496f4d2fdda61040cd5b6591f74e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.7018551209475845, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:38.492862+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7dc9a3c-f11f-423d-9bd0-0aee494b79c8", "result_sha256": "53ec50dc77248a0bb29bd81056c16401de150fa126ee56fb8278e52d30447a32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.837596982019022, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:21.363960+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ba95a5af-6e10-493c-97c2-2e803538ec1b", "result_sha256": "d3ac4a8263f97ab6a0540808be0b1aa525581284d1a6c421f6b70d3bf8df46d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.813024349976331, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "D" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:20:53.566454+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "331a4294-78c9-4ff9-a5b6-a5f3cedc4f1e", "result_sha256": "f0e3853f563bd723763a9ae17e25a1b0fb929fdc30f6bedbeaf79daf6ed134ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.8156198459910229, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:02.800036+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4fe30b32-a450-4207-b2f9-cf04eb15f1a3", "result_sha256": "fc02e297ab3b63e39836255eae38b6f98d71179967fc8bc9be6082cfe04708a8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.0618057959945872, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:28.866608+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a26f928e-deb2-4683-990e-ec8bb0e9e2d9", "result_sha256": "4c4e9f41f33a5e2bb310fea2ea7a3313d2c464b40cef5387a76f7ff87d17e615", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 3.014348943019286, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:07.089711+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "23048935-5fb2-40aa-9818-5d28a6231c2b", "result_sha256": "bf9774cf5098e46085386935bd7f5c59243dda4f63e99403e51701f3a2bc989a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.047521526925266, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:07.701331+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "151bc7bc-d50a-4770-9cf6-ec0efb4712ac", "result_sha256": "3b10dca615b7de6a6d7640b0343fccab4ca210380ae53dc7d0ac571ba6c4ba6e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 1.648776601999998, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:37.403833+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "abb8c655-8b49-4e0f-8371-57badbe58331", "result_sha256": "68746b97b40c896c10c2ca398b2d372ea4741bf0941cde73a70f04e84b73d64a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02488, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2418, "key_errors": null, "latency_sec": 2.683138520922512, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:58.901707+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d31a62ed-71ed-443b-9715-5d67f34c15b5", "result_sha256": "836a43ea36edb8185eff65215f39730db37f5c3e928240ea7f51e2250f903ec2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.7337882320862263, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:50.968831+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8ff4e14-08fd-4bbe-8968-538ffa8e2c27", "result_sha256": "bfd451e1668ca5b8c67ec8d69d16cdcab1116ea8418b52ddcc3a439517584d77", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.1659054300980642, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:30.967254+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "baecb0de-9ff0-4668-a7ea-617e8ca0f269", "result_sha256": "3528577afe04551c094bd5d12d24c47e7be52d5b084cae880780262e3204e1c9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.011195863946341, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:17.628391+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "aed461ec-e650-4e64-8af4-61b3e42cb331", "result_sha256": "708826fce4c6ba4959de73a1058379c2fe951ba48b060f2adb6c2fb808899c89", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.8937858159188181, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:27.085568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "bd1a46a5-bb4c-408f-9de3-af1db331185a", "result_sha256": "c79d02d8faf0171fa7eeabab0a1ef3cdaff2d307bf640befd68f449dd58c971b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.9766826800769195, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:25.636128+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a31ee814-e8a9-4f8e-9058-421bfe2b3de5", "result_sha256": "5de8f735217e5d07d23a7565981e3cc340852a74db7a4833320f6ceb059428b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 2.147802817984484, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:33.152927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7d4a9199-6bd8-4ecf-b7c3-f18b2d49773e", "result_sha256": "0e4639d6992248d1187218dc7f9f8db8b58d76fcb7bbca812c0be34b70bd7cec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.6750179930822924, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:07.058846+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e8917004-4ef7-495e-b145-05de66191473", "result_sha256": "9657e231ba290b9174276e5ea6eb74e8bfa22b8f72284b98904ac06c80a36da0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.5985565350856632, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:47.384993+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9a08e8aa-c23c-4668-ba38-6a1670bc45de", "result_sha256": "6f671dd62448faa74f61c782023bae51580998f961624fabb3d0c2f6af0d5f5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 3.337378764990717, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:00.933371+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "64d42d73-bdd8-4003-8e7a-bc28758c565f", "result_sha256": "1b3d0806e2a75434dcfd3599d40cd465cfc461dab7c0b2fb3e4d12b7db723dfe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.7409636679803953, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:08.919689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ae3d449d-b1f1-48c9-89b8-61d3bac67059", "result_sha256": "0d53df841574e31e46e2f4bcbc5ba730c7130458180db2d49101235bc554019f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 1.7649473830824718, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:46.352231+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "13d610ae-68bf-499c-80c9-3f6d71a9a39c", "result_sha256": "a65b32c4bec4d897f61168587fe07ae0ca99067107969a24afab9937483239cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02436, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2366, "key_errors": null, "latency_sec": 3.479916153009981, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:59.268546+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e8bd2b53-be35-4608-b594-2a54988f2e41", "result_sha256": "eeb7570dc86b0960b6e8d24a187f761fc4100f8af0a18dd95e3847403e06e7be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7975643000099808, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:48.674311+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "98ef7064-1d39-41a1-a6f6-f96b46c5fc78", "result_sha256": "ea10a111c7221c40cbb6d10c4bd698f0b02fd11bbd364f846d2ca1d55cc74575", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 3.006688574911095, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:14.963677+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "52a1543e-3136-4c66-81c9-153b6a1fd608", "result_sha256": "5fb20dda9638333d8f373222bc3dabdade5a9e32a97327ab57357fb31566a1b3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 4.956211532000452, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:06.764863+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "0e2e88a1-0749-4640-a1bf-242bab8286c1", "result_sha256": "43f81f00dac9ea9c91d1f0e1d964b3c3a7da428963a5f4b56928c1ad8b90ca1a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.950238851015456, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:08.189195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "321734dc-bc41-44ab-bcc4-8b59b96572a7", "result_sha256": "5194650672f6d37d15370b435e398e6d6b42b590ead7391347a9bc0c6b5b734b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.3289480740204453, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:21.290813+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c547ef5-b4aa-42eb-a66c-5e637af22649", "result_sha256": "ff1bac31f0e4245b579ada13045fe5dc4d615020788837862c573f4b0d386e75", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.029341942979954, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:12.298287+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4fe67eac-7977-4e28-a31a-47faf23173f9", "result_sha256": "e5e4f35a8dd066e80a4eb479c04f62a0abd96f0cf1f2d5fc373c479c5b0c5538", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7646728709805757, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:40.890245+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c0c65e79-ff53-4be5-bc38-4f410aae830a", "result_sha256": "87889422a14b977accb3ab8468fbbbad65b4a7a9c4e2a9503e8b3decc1b0abb5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.4017998420167714, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:50.623313+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "25ceed01-1977-46f1-9a1e-39f3e9dfafbb", "result_sha256": "00fc8f3371763032f1b35ecf302c3c4623c02a40c3aa9c107254361fa15abc5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.746525185997598, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:25.387469+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "efb9129c-1885-4557-9354-3495bc3fefce", "result_sha256": "c19637e4ece2764d80b50d53fc41fa01640f72fc6a88fff52536cf69b31869a2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.6452339300885797, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:26.395028+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "26c5ac38-ebdc-40fe-876a-56f7a19ca7ad", "result_sha256": "eab17c491dd7cab9f45b67629c1cee349a7ee5d31a0d962a42bd64bc3615464b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.6178258300060406, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:50.959372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "bcd33b21-79ea-4b64-b985-6a55a451857a", "result_sha256": "b9f3c20f5ef31192e64306976e3ea68dd84da1b0a3a6d802e778b9ad5387f5ce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.8864852610277012, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:36.190656+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f1fdf8f8-88f4-4dcb-ac2e-9a48dbb5e68d", "result_sha256": "c67db6405993244cbfceb8964820c0dbee4b01ccb20a7ecb12e7919d250b8342", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9373363140039146, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:09.885214+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1c1da9b-5238-41c0-a417-be9f6b9fd767", "result_sha256": "f2f4743583c03bba072862e9adbf328ab5cbd0e88fe901e04a76cc276c1cb8f1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7003113819519058, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:36.104520+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d930f327-b476-4553-8909-3b5188badbd7", "result_sha256": "1210b1aac0c13628704dbb795c5bec9a23fd43023a27193709fdf816b04b3cbf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.123223770991899, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:07.055947+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "184243e6-9878-4d97-bb89-2d6055668941", "result_sha256": "da13a103a577f59b791b3ecf39956f8e2f877e507ecac993e0546cae5254c588", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.1061713750241324, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:18.340367+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "09971ea1-be0e-4bf1-bd64-309a3faa645f", "result_sha256": "8a1c9d95914012ecc0b30d7e50cfc7fdfdb1203580a3dd7cf0537f740036d198", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 3.040577891981229, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:18.641372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f18cc55c-7554-4aef-95bd-5371a8e7056b", "result_sha256": "02d55c6a840c93c13cee33c4d36d856147e4120e475c7ad4503e500ac79b3589", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 2.1516727240523323, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:43.420614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c307f21f-8255-4f61-8290-586cb3720434", "result_sha256": "48baf0dd65133d566b6473d52092584e7eeffebf1a69cbb16c779ec86615fd6e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.9021673829993233, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:56.074669+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "040e3398-bc50-42ac-9888-92cc1e33d2a6", "result_sha256": "e2c3caa9cc5605e0a150d0ac08ab494004ef6f11340fa1f85bd4e2bb985cf8db", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 3.19808132504113, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:15.312383+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "9ed3d3a5-b611-4b71-b244-c497c2ee6b3d", "result_sha256": "aded73b5e18cbb98c1d68bce0a2afa3ad9be1f23bdb97fc1f8f231e3313a23b4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7805279919411987, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:20.746599+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ddd37702-3320-40ca-a8a0-c7b9bd7332cb", "result_sha256": "6065602f3e27fb31a1a53538db5dd37fc04c9efdd87bb762c667b38df2312bbc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7806406879099086, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:36.166663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "bc66fe62-fe5d-47a3-a7fd-26617eedf694", "result_sha256": "b633cc7059ef2395548ac2893587a3c5f7a93dee27e04d0c49fc47762cff6cd8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.7723401909461245, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:25.552651+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ee984ffe-a4b1-41be-85f6-0240043a9983", "result_sha256": "0b0704da4a596b5232d914f6a36805f8092161d3301ae13166fbae5144467966", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02457, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2387, "key_errors": null, "latency_sec": 1.5877959999488667, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:23.471229+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "cfaa5e01-854a-4527-8d72-2c834b6b67cc", "result_sha256": "2e5032f6b832efb85d7c9f749273b4695a7cd1d797185ae0fe94f1befc2960be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.6781330050434917, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:37.187091+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3f180ce3-9c4c-4e19-91ac-6ec71ac77d34", "result_sha256": "93be9579701cc59cc801946744e14315226113cfe750409254593569ce9ef391", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.9710747170029208, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:41.411370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "333f2ffa-b3ef-4395-816b-2f21fa593f6b", "result_sha256": "8f4d7d17d6071db403c26db8d19be2ed89e28078456d40e72887440507fde5c3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.5934550740057603, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:00.361705+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ee36f1fc-0cff-44ea-912d-cea82740976e", "result_sha256": "634c4e8984d38860e8042d42c9df72b8b56d845f6592bad1b68675a0221825f2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.366172331967391, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:03.256440+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "534eb7ce-f523-42e3-95bf-00ec9323916c", "result_sha256": "0694eb024431575c3ca6bfe13752040249e298cab2abc7a6bdf2a6e1d6bc860b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.6246223950292915, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:14.089856+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a525ad88-9e9e-4feb-b6a4-60d3681522c3", "result_sha256": "14c2949426ef2210c862cf0fca003752244f728617b4366b4b84e28de805e896", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.7456459649838507, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:20.417281+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "36ebe72b-0502-4ef8-8185-6f2e11cdcc81", "result_sha256": "f5dc001ef50c9530c1504da1440975f83e8e93952a14b91d47e508d136d29f17", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.9162903079995885, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:02.005667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "173eeddf-8dc9-4bfb-9c50-73e4e1b61a08", "result_sha256": "785497f1544ecf0ed95e6741e3915c296508241914f1a21ea8c12e33d8c03ec8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.881093348027207, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:18.107518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "bfadfc82-17f3-47f6-93ba-deb77208dae8", "result_sha256": "6ad6db613f8352296405106c19adaee2170a0a9dd284e94e4230bdecbd60d6d6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.664266107021831, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:59.333426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "df4d32ee-534d-497a-b104-c2e9707c4208", "result_sha256": "f3111e24b754d701798dc12c6fde18d61951def7d280fea296052e2992a84c98", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 1.7649094460066408, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:32.971030+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "02d83469-7759-4f77-9216-cfa5f63792f3", "result_sha256": "473020b9ea089f649f6648c1d6f67273951fa64eac5b7a49f903f795b30d89c0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 2.8044826090335846, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:43.669969+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d488ff80-a38e-4522-8945-86dfa8f2f64c", "result_sha256": "6527432d60e6482777ed379ffd737be6021374fef1da35a6d8a31f8ac5d742b4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0245, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2380, "key_errors": null, "latency_sec": 3.2275245611090213, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:16.447533+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c9cf2fe2-d862-4aee-9664-138fda2b581d", "result_sha256": "5ecc6152b30108452f214fa5cc9cc0b69d6131ee555ef621781a9ef33a5a8ef0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.8308793450705707, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:57.589351+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d6e4aa2b-17d5-4fa0-b778-092535e7728a", "result_sha256": "0bc534fb0ca99320fc2f779676c1ed1d8bd31db3ccdd56d1fbe72a44376b9d16", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.7522210900206119, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:32.467621+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7cea6c09-37d2-482c-984b-f551534f34c9", "result_sha256": "917aded9387cc808b924076b47b253b1e9d84e49e54d91f399112d0b25103599", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.8250469589838758, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:58.383651+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "5682f804-0a2c-46b6-b157-e2bef6adfc1d", "result_sha256": "a98476b803f8b1792a16415fcf37ef91e8ceeb44772a82a957ec78347b6bea2a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 3.250095516908914, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:35.392474+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "e289d4ee-873c-47f1-8cf6-f76e9f66a258", "result_sha256": "7825a97152f45dbf4a9444d2e280324dd5a605624ab862a7ef57d165507baecc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.7095739740179852, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:44.210414+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "656d3440-79db-408d-b855-2794839c87ef", "result_sha256": "7d7683d28b6052d548601011fe5a4e936a888ca586bfc83e09d45cb865fbaa22", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.8361779289552942, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:40.217076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce117321-d21b-47b0-9d08-8127feee9896", "result_sha256": "c67b1ee499e80234914467126c8f141445f888ed4d11162ff7ef1b91368ac639", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.4367002960061654, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:52.530566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "c234f1f9-b61f-4d3a-87f8-b10268a1fec4", "result_sha256": "143e1d3d2fa61f6dd09e207279208ff373654738acb4256f16db8eac804cc392", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.8348010319750756, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:34.825217+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7fae146a-39d9-4106-9e16-eb93a6ead748", "result_sha256": "85137dafe808d29742e483c31ae01a84885eacfa14be68f2360ff34c13b7b33d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.846642707940191, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:07.537783+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d32100ec-3fc1-4f23-8465-3d019136a0d8", "result_sha256": "08798179943de76a607cb59c387f856e4110beba0f2b3482b4bdafff7dd64057", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 2.2669507779646665, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:56.124629+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "7208ebed-349d-42c9-9d9d-e07246e232a2", "result_sha256": "4b2d5ee2e2925df5f27c2c3675a44b083427935fbce53c3a95df38d06c707609", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.7538605600129813, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:41.512129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "d6190e2a-ad43-4c00-82b4-c94d3b8a9850", "result_sha256": "a8f7be7378fe4c53543a5f920530b6071cd6d4f063d119b02883f52d9c6aa239", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02459, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2389, "key_errors": null, "latency_sec": 1.927152162999846, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:51.981262+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "a4a56a35-fed6-4655-8dbf-1302173afbe7", "result_sha256": "56412fcd3962d333bc124ea9965324add1c9b3f7a1dc72b52d33c49aa5fc571f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.6275190289597958, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:05.791489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce723dda-ee0a-4220-94e1-4ce5764b73bf", "result_sha256": "483ad631487a884ee8ecbd735ba550516929910470c713d98badb354fb20790f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.8257894199341536, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:32.420351+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "84b92b37-51b3-4f9a-bca0-16a21b890f8e", "result_sha256": "bd20523490ddf8139ceecb2bb133b766316e6c28c9ece23c8a440b15949605de", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.8433062949916348, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:50.037840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5f1efee-3807-4d13-9b47-fbd7138f62e0", "result_sha256": "0aad9d9874b593c876bb3ef00bf01ddb869d3f570e741538ced30d5aa0c31e0a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.6721308049745858, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:10.272370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "46ad9117-0064-4dea-a828-84f2a65faf95", "result_sha256": "8f15253e903c0a5fde282a711819d45c5b7d069409572020cf1dfdfc9416ce67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.799745506956242, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:03.220667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "4ccea91a-2e32-4642-91c7-5eca76f00f5e", "result_sha256": "8b4eac0891f2ea0886f95b16d3823341a14cf652cad6b2b414e59ff7f1eee285", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.70649367000442, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:50.979860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "3f96402e-7be0-460c-8400-a55d5086204c", "result_sha256": "f63a9e708b3e7db20c3813a9b7836cdd762c2283bc4b0418fc4fff13048e0113", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.7762822039658204, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:18.073180+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "47f632f8-d4a6-4601-8010-1a89e23004a4", "result_sha256": "12d6f621275229a158acc01febf1e68f9a08a688b9078fc79d0ca0ef451c76ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 3.9905719599919394, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:07.862785+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b32794d-5ae6-4ca7-b45f-4abb9a5ce171", "result_sha256": "76392145d0610e03f6b4bda1eb33da329476e8784d190cc55a41554e5b8f856e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.8439021781086922, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:48.975580+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5af7be7-0bdd-451f-94fc-0558e15781dc", "result_sha256": "7c87a76c4b67be1bf48f177399878658e6d48a94c08138fbf2cb7d0f0e9388bd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 2.052714627003297, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:31.187951+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "999320f6-c577-4389-9f7c-c77bdf1fecdf", "result_sha256": "989026048d75a7da74410cbd05a37266d122777435b739b51dd630e9786e4525", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.7462361629586667, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:55.388700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "gpt-6-astra", "status": "final", "included_in_comparison": true, "source_attempt_id": "664995d6-638c-4f3e-87f4-d74a7ee22429", "result_sha256": "75023ebf736c2a2143149a24f1fdeadb15d5fa343d30e2c38037eea6fa6bd7da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02437, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2367, "key_errors": null, "latency_sec": 1.6377637879922986, "model_name": "gpt-6-astra", "output_tokens": 14, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:42.273976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "065c8000-9b47-4522-b537-6fd057535364", "result_sha256": "8e1120d1911623696430c70026e7a95942c90dd9b32346629a1ca3d6038db0bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00415, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.096631825901568, "model_name": "muse-spark-1.2", "output_tokens": 230, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:50.537193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "06fa58a6-88c5-4f08-9d8a-1fae77dc224c", "result_sha256": "c5119e186182858184ae49418de9da622f6fb9394d54bef322cf8526d151fd84", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00479175, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.5323234780225903, "model_name": "muse-spark-1.2", "output_tokens": 381, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:17.763617+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "5c79defb-b02d-403e-8b50-4c03688620a2", "result_sha256": "42a958731dd6bc198975b508c965c99c4cc83aca6b7046663eb6c3350bc38723", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00456225, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.1803069489542395, "model_name": "muse-spark-1.2", "output_tokens": 327, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:11.987854+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b2de895-31d7-4d15-96bf-dd5a59801eed", "result_sha256": "2288f445fa6a96ede2c32dc2942caecbb793704668a4f5d9a3d38ec4a9f1c1c9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00458775, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.7344858230790123, "model_name": "muse-spark-1.2", "output_tokens": 333, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:15.108012+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "bbee5cde-2cc7-45c5-b67b-8dc848cc7ece", "result_sha256": "1d01d9d35556ae58f527b4fc2f487d922a0b04e2951f599208c13e4aaacd32d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00466, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.206489242031239, "model_name": "muse-spark-1.2", "output_tokens": 350, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:41.163548+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "76a2b944-a1c8-44ce-9473-764cb9749035", "result_sha256": "e949f043fb761275077016e833f818c9893c8a70b941ed054652bf80072dcb82", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0046685, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.121943050995469, "model_name": "muse-spark-1.2", "output_tokens": 352, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:56.724458+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "08b5d65f-ea91-4af0-8c56-c41e1ea95e76", "result_sha256": "d8842943d02b6ec1d5fc09551d1df2ee24fdadf79a64377572bf25212d6110d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0060115, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.5162623560754582, "model_name": "muse-spark-1.2", "output_tokens": 668, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:42.613869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "33da9a79-dddc-4920-b72c-323a5cfb8a70", "result_sha256": "3ce64cdb31c9049a1f1630651d7a0acb0b4fbdaf7c45627bc8fb4bcbb852f878", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00518275, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.252273533027619, "model_name": "muse-spark-1.2", "output_tokens": 473, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:01.148798+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "6e686f2c-94a7-43d0-98e1-7ddf31ebfdca", "result_sha256": "7d6642246b2e26ae31321509a8acdf53d48b7826bedc93f917dabf470956ad27", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00429875, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.5358600839972496, "model_name": "muse-spark-1.2", "output_tokens": 265, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:32.371798+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "335f6d66-d80b-4ccb-9753-1e3d5634dd82", "result_sha256": "0d439d8f2f13e22f01d59d24ac72cb854e7f000bb4cb2827ff8831422ae9d4be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00442625, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 1.8900927560171112, "model_name": "muse-spark-1.2", "output_tokens": 295, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:54.480988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "3efdaa4b-5cc1-4022-b59b-374d37ad56e2", "result_sha256": "a6e05d74f1a8d7f21df0ccf4fcc7e4516f815b01547cadd898f496229b105855", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0043625, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 1.8104202680988237, "model_name": "muse-spark-1.2", "output_tokens": 280, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:31.567450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "281f7885-1168-41fa-9007-a86043e9ceaf", "result_sha256": "f5088c5d164673188321432f200022fe897fd09cddb402dcb7e416b669217ee5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00438375, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 2.198025421006605, "model_name": "muse-spark-1.2", "output_tokens": 285, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:53.544822+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "dffd9a69-984c-4bcc-bef0-f56ea3e7ad5a", "result_sha256": "a61d516d97142a0eb3944a19402897a8908c728e4f49418488c9b9a9b50800ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00510925, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.894737556925975, "model_name": "muse-spark-1.2", "output_tokens": 436, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:58.646978+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e27b8c69-578c-4f2a-a0fb-f9eab870db34", "result_sha256": "b8bf84b1faa4804a38fb83a6ffd7e9dd4e9a817ab1a7105fa944bccf712e6a82", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00504125, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.4593563420930877, "model_name": "muse-spark-1.2", "output_tokens": 420, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:03.107716+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "0120f62b-2572-47bc-8aef-b77e087ee661", "result_sha256": "0109606a6d3354681e85e81d4be8ad23c5130711b9ecb3dfd6a31a67bdbb9184", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00525375, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.7834733520867303, "model_name": "muse-spark-1.2", "output_tokens": 470, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:02.922364+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d7109301-b4c0-4e93-9023-2a1fad8c8229", "result_sha256": "12a3529112b3491cc0429f6bf0f28f6d4542840cb560ab8da30fc77e56adf686", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00504125, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.09617640695069, "model_name": "muse-spark-1.2", "output_tokens": 420, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:08.103833+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "6aa5897d-a591-4065-8fee-4fdafffb8107", "result_sha256": "cffd13c93e9310c3bcd8dc2b44cc9e839b6710469ca38deffac69d3d0b1155db", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00754025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 6.342727012000978, "model_name": "muse-spark-1.2", "output_tokens": 1008, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:18.047119+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "66f1c9d5-797b-491a-b998-de788ba28b56", "result_sha256": "e6d04f0619b057bcb6d4fc22d8f3f004b59573a5791a1be7b7bd804fcd28b543", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00475225, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.306712856981903, "model_name": "muse-spark-1.2", "output_tokens": 352, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:22:15.760083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b6a08cc4-2544-434e-8845-2d0c6c0a7e86", "result_sha256": "91e3b5c51c227e6aa64b10c1fa729a71beab2598a0fd8637e5414fdb338b1e6a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077995, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 5.970136805088259, "model_name": "muse-spark-1.2", "output_tokens": 1069, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:48.898191+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2ce611d-24b7-4e92-a786-ce82e93d4a4d", "result_sha256": "ad52ce506dd1152ed7138e2323538296c61081fef818b7acc865a87df99a9609", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00454825, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.188646031077951, "model_name": "muse-spark-1.2", "output_tokens": 304, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:44.781853+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "19115295-5ba3-4345-bc28-b2cd8da7359f", "result_sha256": "80f9a57b3ff2de23949e4fa3ff8a3d70241f664a1159cc467874236cb3ab6ec4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051475, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.633492987952195, "model_name": "muse-spark-1.2", "output_tokens": 445, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:29.433098+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "87646e74-3af3-4c0c-9050-96e1203f7bfa", "result_sha256": "3c2419ee119444ae313f0cfa91bf2aaeca92b691ab52ed7ecde0a4a9f2b4d7d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00502, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.7796575740212575, "model_name": "muse-spark-1.2", "output_tokens": 415, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:24.279823+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "6ab6b10e-6a56-4f1e-8f96-d88dd86670a6", "result_sha256": "7d3937f979580df78d2567e4c5514a7f8a508034e5aad291c28a870bf18660b3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00490525, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 3.3230832429835573, "model_name": "muse-spark-1.2", "output_tokens": 388, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:10.389536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8abc1c14-0e26-412d-8127-7edbb99f9cfe", "result_sha256": "f6f4ced196805db926580a60cde8436409d8a492eba4c213b4aa2bbe6d9cb73a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00465025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 2.514648661017418, "model_name": "muse-spark-1.2", "output_tokens": 328, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:35.048264+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "97714ec5-d959-4b33-a918-d16ad2a2ca4d", "result_sha256": "5a4e157b2d3ba52559f87be2ab9a850d9bb0b64798bb6969c2c4b218655ca59c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005258, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 3.327465069014579, "model_name": "muse-spark-1.2", "output_tokens": 471, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:54.528527+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "834d68c8-7880-4344-a6f5-61cdb68b9ef6", "result_sha256": "04dd27b8286c98c93d41403bbcf4d6fcb96ab0cc7d42c7169e0c1fa44c775b55", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0058785, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.086923344992101, "model_name": "muse-spark-1.2", "output_tokens": 617, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:06.369704+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "664df66d-1432-48e9-a323-00069b591a0e", "result_sha256": "74195a3b593dbd82d2264cb2791c785d8c5487d8f72dafdcb06566b4addb446d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00466725, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 1.7913931099465117, "model_name": "muse-spark-1.2", "output_tokens": 332, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:58.768975+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "630324c1-b2b9-4e2b-8123-ec4ec6f20078", "result_sha256": "e5e6ff5f0e1d4276f7a323e6a95da598e65241fba91f5d4c407c57797fd69baf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009491, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 8.433768459013663, "model_name": "muse-spark-1.2", "output_tokens": 1467, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:25.958319+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "88468193-8a29-4369-a77d-b11099661343", "result_sha256": "b417dccf9c8969241d65aac276a324e3b0c5a2046cc968a13518789465f7fa81", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00366325, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 2.2098521419102326, "model_name": "muse-spark-1.2", "output_tokens": 124, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:42.193689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1677a9c7-bb7a-471a-9ca2-93efff68601b", "result_sha256": "5134830884a5b42864a3952d91eb21819c4889dab1b53bb7e53022315bde1141", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00372275, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.002737027942203, "model_name": "muse-spark-1.2", "output_tokens": 138, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:11.778202+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "0e52fed9-668b-4f59-8a3e-1ae788daeaa8", "result_sha256": "c9bf8d3680234a6564a642983ac88de2458474cffec37758af4e170ab563644d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00379075, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.6444625149015337, "model_name": "muse-spark-1.2", "output_tokens": 154, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:30.863437+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2f6a01b1-27be-40a6-a13d-510902c76075", "result_sha256": "e2804c34af07746e5725d0055aceeab9193d4274d85610f1d83bec408c13814a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00413075, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.3316422390053049, "model_name": "muse-spark-1.2", "output_tokens": 234, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:03.167904+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e141118d-2bb2-4b0e-b51d-85d9d6f4fd96", "result_sha256": "b76277fb89c06fc6dc00ea32cb9aec6c07002e4e47f6883ce559958aa53fd906", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003914, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.0989621100015938, "model_name": "muse-spark-1.2", "output_tokens": 183, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:33.538382+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa1483f3-d00b-4e52-ae79-6100dffb87ef", "result_sha256": "1645b8548c8e99c0cc16836b072a7c1cf7c9acdc87e7f433363619ee94d66746", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00390975, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.7206156300380826, "model_name": "muse-spark-1.2", "output_tokens": 182, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:01.875675+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "fede20b6-2751-44da-82c2-fe93bf7de6a1", "result_sha256": "e0196d2b641c0227827007bc5202179fdf4bb4f3b1ded175cde42bce63118153", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004067, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.7994764050235972, "model_name": "muse-spark-1.2", "output_tokens": 219, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:42.594610+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "59ef29b3-8fa6-4602-b604-3482f2b9c99d", "result_sha256": "950ad88b6689fab455e81f9dbaffab42ff70d28870d87ac2c0809cca6b815419", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00362925, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.4851508410647511, "model_name": "muse-spark-1.2", "output_tokens": 116, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:18.266653+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "3a23150a-1971-411a-8309-678b53cb4a6e", "result_sha256": "68945ec67c049bac732c59cba9ec36b636a1b67e400a4bc5e5b964e4169d7cda", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00779775, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 6.222538274945691, "model_name": "muse-spark-1.2", "output_tokens": 1068, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:24:51.323947+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c711698f-05a9-4276-9724-125979262cd2", "result_sha256": "787954f5a2f7858b707e62c16f487d819413dccdc651108397f8e7db891bf9cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.017152, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 15.383360221982002, "model_name": "muse-spark-1.2", "output_tokens": 3269, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:26:47.915623+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "87d5f08e-853f-4688-b9a2-4e75d5db04d5", "result_sha256": "adc0c8d350f25075859c2d039e5d066fb518375bc13403dbc60e8f3b7dcd610c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00627625, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 3.6589491240447387, "model_name": "muse-spark-1.2", "output_tokens": 710, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:00.025096+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3f2ad9d-b188-44b9-be10-e0278b4e9685", "result_sha256": "8061edd27d22bad52cfec589e36ed3f82d5948634e40ec31209b1d7193b8045f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01375625, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 12.197680026991293, "model_name": "muse-spark-1.2", "output_tokens": 2470, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:08.255818+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0298a5a-0329-4074-88d3-e7a930220a89", "result_sha256": "f1c6a91ab11c848eccea3c147efff091480b1d4d37f5bd96555d3221ca7d83a6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00622525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 4.924019098980352, "model_name": "muse-spark-1.2", "output_tokens": 698, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:12:21.352185+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "98661232-46d8-4ed8-b030-092545bfc8a1", "result_sha256": "6eed451833d0c8533d920aa288a0d215474c622293f9b2cccc133d83238c9c78", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01024575, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.075815274030901, "model_name": "muse-spark-1.2", "output_tokens": 1644, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:11:43.627565+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b75cafc6-1d6d-4dcf-944b-2be8265453d0", "result_sha256": "7b33289cd8cd78cdbe804d7c7e74d6dbbf931fb2d488f7f35a6e0e35698e3db4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006119, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 3.8893353029852733, "model_name": "muse-spark-1.2", "output_tokens": 673, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:29.692857+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac4f48b5-ad8c-4677-8083-232e54f6dffc", "result_sha256": "7d75f6cec5aa4923414b4a7df9ec0ccbcc0059b7671be750f92e90017c06e8a1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00904725, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 7.377113669062965, "model_name": "muse-spark-1.2", "output_tokens": 1362, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:19:23.617062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b5960e3-6436-4dee-895c-835009c330cf", "result_sha256": "36ca400e59a80f35e6b79e69b70d5146cb06a52172d9c02c31fab6f3b5b9bd63", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008329, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 6.723455403931439, "model_name": "muse-spark-1.2", "output_tokens": 1193, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:30.621290+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "6226427b-c04a-452e-b8c6-566f2f8b8786", "result_sha256": "7ed12b6f37cf984a241213cb9a8d739ac96619079d4668f8481b037af43d1122", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00753, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 4.5226199850440025, "model_name": "muse-spark-1.2", "output_tokens": 1005, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:15:27.705044+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "528bbbce-f291-4bcb-b460-2ad21c8b6817", "result_sha256": "d5f9b6931170d40583343b38da619550e6aba73f92a37ca925778cbe9a5fcc37", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00900475, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 7.724259817041457, "model_name": "muse-spark-1.2", "output_tokens": 1352, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:57.024577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b03045cb-d3b3-4f37-89f5-58ee91a1f238", "result_sha256": "ee7ae70ae01f9d1feef5c26a458fe5ffcb4cd9c2b6771bcf184a34e73af08f64", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.016302, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 14.555025930982083, "model_name": "muse-spark-1.2", "output_tokens": 3069, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:26:08.986475+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "888b55a5-0fe6-4361-a933-155dca012050", "result_sha256": "50b947c395a739b2f4cac5a51bb583fed5743bb6dd1a4c897da067dcf127c66e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006527, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 3.7681956259766594, "model_name": "muse-spark-1.2", "output_tokens": 769, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-17T03:26:53.728441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "6a577773-511b-4367-91da-6466cf18908d", "result_sha256": "499372bf8b35c0aa97cb3d616b0ca441bf65f79cb3ec9c13bb0c707fa80c9497", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01464025, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 15.955028312979266, "model_name": "muse-spark-1.2", "output_tokens": 2678, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:03.767819+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "ce2461d4-b8a6-4187-b6c3-2017e11ec552", "result_sha256": "038e82984f7a8e48b9a1485330c9df9b43f58526c0c0b80e2c75ebac51045449", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00656525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 4.336979693965986, "model_name": "muse-spark-1.2", "output_tokens": 778, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:40.263066+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f14e4acf-f21e-458f-847c-bd7a71c3009e", "result_sha256": "38b7b7275829bfdbb8c385e2b565e9f69db41be56878dfe764ca2e7d359a29c6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00745775, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 5.708480152999982, "model_name": "muse-spark-1.2", "output_tokens": 988, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:29.142405+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8bf69654-eaa4-446d-b854-065deca7592c", "result_sha256": "b3f8759d27f2e5f0b72eb6f338fa968a6a6525771a2fc094244d48544b00aac4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01185225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.973216101992875, "model_name": "muse-spark-1.2", "output_tokens": 2022, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:23:51.081230+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "11cc57b7-23ff-4eb3-a61d-20defdce621e", "result_sha256": "f98f89adb51aee7902a1495c437a6ae42b763f924f0242bfea8ff867ab45935e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.019753, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 22.514183121034876, "model_name": "muse-spark-1.2", "output_tokens": 3881, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:47.838835+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "50464778-6386-488d-91ea-fe85b8671b9d", "result_sha256": "08367ce72fef5efc656666d75a72784efccb0154b098eb6fdf5e6dffc5671b7f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070115, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 5.286906089051627, "model_name": "muse-spark-1.2", "output_tokens": 883, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:17:07.488966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "009ef5dd-53f3-4b6f-82ed-f4b091ef8721", "result_sha256": "7976f4fa092db489c17501a9329131042713ff741ce8fb2a248c250e8dc6c04f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01399425, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 16.890427712001838, "model_name": "muse-spark-1.2", "output_tokens": 2526, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-17T03:21:20.763861+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "646c5bc5-df7d-42ab-96ef-fcb0809bf374", "result_sha256": "5c0cc480e3d0fea5914adfa3636ad9659a1c0216a5f2ec50b72ede2e6cbdeddd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01141875, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.537435580044985, "model_name": "muse-spark-1.2", "output_tokens": 1920, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:07.916065+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b166ce0b-202b-4681-9694-0a30a765a8b7", "result_sha256": "39e00fa6541a2b432b3cd938c13d0050dfc4879b7fd018eb126d7c20a1a3de0b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01270225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 12.001281325006858, "model_name": "muse-spark-1.2", "output_tokens": 2222, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:19.750902+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "82d5187f-90c2-4441-8c5b-540eb85cae2b", "result_sha256": "f15143f85b8d8262303fea6a38750a790d252f0d4375ef2ad28a5f16b07d8c27", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01114675, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.147914691013284, "model_name": "muse-spark-1.2", "output_tokens": 1856, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:43.839308+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b7ffbf56-456b-473d-94a3-fce7980d9df1", "result_sha256": "4b05985a46f62045cd57d4389f7cc69b2bbb74b83a0114a163073aacbb0b0507", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02144025, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 22.674961029086262, "model_name": "muse-spark-1.2", "output_tokens": 4278, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:17.509642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "438756f0-c7da-4033-9286-e398b184ce27", "result_sha256": "183313921b76b45adbd17824038ce92485e658831c3279bd47d838ed93dc8ed2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0058005, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 3.6018577340291813, "model_name": "muse-spark-1.2", "output_tokens": 621, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:13.611377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "7d9362a5-234a-4f45-9f10-4cf020907d5b", "result_sha256": "bdf954b827c5460979cc4fede466f77542f074c792e7512eb8b403e8ce038591", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00529475, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 3.0216475769411772, "model_name": "muse-spark-1.2", "output_tokens": 502, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:56.088074+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c78b12e9-2eaa-49b7-8e42-970f2e0eee3f", "result_sha256": "64d5b92738f45814a936301d25a1bf4132a393d02186b92625458616af70473f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00438525, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 1.8540575100341812, "model_name": "muse-spark-1.2", "output_tokens": 288, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:29.130556+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "55e009fb-537a-4cc3-b440-c2938b2bce2d", "result_sha256": "252dbdd9b420eeefb2a0036613dd031808ece3212a569b508657fb14c95c039e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00761525, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 5.943759222980589, "model_name": "muse-spark-1.2", "output_tokens": 1048, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:59.061666+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "59e6c1d2-ed51-4342-bd2d-21d529cb0bfd", "result_sha256": "a90915d6afd6bc1b2061dc078091c606f1613daa1f3f4396f4df31fa4391e04d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00482725, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 2.4445153640117496, "model_name": "muse-spark-1.2", "output_tokens": 392, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:03.220993+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "0553e7e0-595a-4969-8686-6a4bbcec077a", "result_sha256": "93980313a95da81b830eca5443238d66b99986217a20806b6c15fdc2256658aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00570275, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 3.6394311049953103, "model_name": "muse-spark-1.2", "output_tokens": 598, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:03.774713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1a64d6c6-c83e-434a-8dca-dbabf5c7a8f6", "result_sha256": "7f9674d2ebd6ea67dc2c88b5e86cdebe83538165dd505c267f9fc687c293d1b0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00490375, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 2.8059691989328712, "model_name": "muse-spark-1.2", "output_tokens": 410, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:21.316702+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b76cd85-89ff-4b71-83fc-239bd58fc317", "result_sha256": "7f8f8cfea6ed37c7d7d0801294c2a534cc535258f241a1ce93aae305abfd48fb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004823, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 2.401786943897605, "model_name": "muse-spark-1.2", "output_tokens": 391, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:37.775223+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e436400b-8b4e-4e46-8916-4f20e0cffc9f", "result_sha256": "41eb679907cb8cfc9966f1c4674eb51aa5c212cdeb2be88e50572937b3bae6fc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01092175, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 11.991850486025214, "model_name": "muse-spark-1.2", "output_tokens": 1826, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:18.913754+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "00718834-abdc-43f7-835a-ba6c5155b921", "result_sha256": "29f9c832ed073ec5dd009726deb30323e4618bb0a2f8e3604ca9422480ac55fb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02356975, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 28.28650942898821, "model_name": "muse-spark-1.2", "output_tokens": 4802, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:02.201953+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "975392d3-487c-49ae-8d91-ec20c0d0b52c", "result_sha256": "9376778b1ad6701af0edce3861bf70fcfbd8b0401fb71cc3e2b55a448722efdc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01071775, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 10.164172252058052, "model_name": "muse-spark-1.2", "output_tokens": 1778, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:28.866906+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "466995f8-28d1-4a31-8ab0-d94d40a40a02", "result_sha256": "d31496a4f4a75cffcd2a6ed50a7afd055640dc4683de53ff4c81705f6932014a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01128725, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 12.052449577953666, "model_name": "muse-spark-1.2", "output_tokens": 1912, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:10.059027+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c02c3efa-a3ba-46f3-b1df-945b1fd00646", "result_sha256": "911e20ceddd63afaa4b018c7baa08d469b73353393abb13df775997f2d41081e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00384, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.5651538070524111, "model_name": "muse-spark-1.2", "output_tokens": 165, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:10.277551+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "72b33635-b156-48c1-8c7e-e2b02cf7694b", "result_sha256": "24c184c7cb829e2a93633e2023cca6a833b0f1fa78d29bc3e96fba7c0a4e701b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00389525, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.2191652540350333, "model_name": "muse-spark-1.2", "output_tokens": 178, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:24.984777+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "77e5a27d-d03d-432c-8461-df4c91203ad8", "result_sha256": "afa8c3f246622d97e63992a619a3b801895542b97f71c76fe92261b9ba674f1d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00370825, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.994903240003623, "model_name": "muse-spark-1.2", "output_tokens": 134, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:12.778705+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "5cde2d15-7288-4463-9056-9f08b9866bce", "result_sha256": "1d3eece9c6f3341392be5524cc9f4a717bef320295849900c8b6f80e1ef7f6be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00387825, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.2301527929957956, "model_name": "muse-spark-1.2", "output_tokens": 174, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:43.477601+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c45c28e0-69bf-47c3-b11d-aba20f2dcf7c", "result_sha256": "0b4824159a73f6ae6251fe56805da88bfadd0b0d0ddcc96d51a76d411d0982cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038995, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.2975182599620894, "model_name": "muse-spark-1.2", "output_tokens": 179, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:48.119423+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "07498aab-c86b-49f1-a5b2-34b7b5cbdf47", "result_sha256": "07c8abe9115945cb679fe11bc3d850704de5c3869e6905b533ea84514ef7eb4f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003806, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.0448581280652434, "model_name": "muse-spark-1.2", "output_tokens": 157, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:15.387423+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "819b3212-9c8c-4e93-9f02-3e0da882e081", "result_sha256": "9860a66b4d5d608201f54a03a4e2b15c7392de2465daad82cf45ee0ddb05cebe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00393775, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.2942044719820842, "model_name": "muse-spark-1.2", "output_tokens": 188, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:56.338644+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "6907efcc-a052-4faf-b58a-e74c628292a8", "result_sha256": "8a273cc8929a78c97baaa70e7509f9b77c9b4c6cc76131419df503f0190f327b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003755, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.0576571830315515, "model_name": "muse-spark-1.2", "output_tokens": 145, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:00.078295+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "666d7680-910e-4e23-82a9-51729f175c38", "result_sha256": "939c92df562ddc0a67044f87e76f115fc4e672465c781719bdf84fe20500280e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004027, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.5149732339195907, "model_name": "muse-spark-1.2", "output_tokens": 209, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:40.137028+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "4dc17f15-20e4-4fe5-9e38-9b488ac5981c", "result_sha256": "0a5e638d7016633645b48c1701b4f14e1bed4c8e6f03dd54fe86c17d7b36edf1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038315, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.1591214939253405, "model_name": "muse-spark-1.2", "output_tokens": 163, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:30.441813+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "cb05bb9f-cf53-4793-8a0d-493e167359d1", "result_sha256": "924b26931a57df678451726567a79a0c14e1f6dedb2fb0c3ebd13eac42562234", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00405675, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.6657490889774635, "model_name": "muse-spark-1.2", "output_tokens": 216, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:34.711777+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1de09ae-ea3f-4323-b787-6cb43f743a85", "result_sha256": "b7703f716bc0986b7a9004a9aead6bf1a2ef26577566dc05c2bcb564d07c5c41", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003993, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.6066754390485585, "model_name": "muse-spark-1.2", "output_tokens": 201, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:23.601191+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3433c37-b071-4df3-8c0c-3b34126c4fdf", "result_sha256": "af6ff6b76a4321ec0d9dd3b3b5b4fa6a0ebe2cd06aacf8ee690528bb8b85c8c0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01246025, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 13.156258863047697, "model_name": "muse-spark-1.2", "output_tokens": 2188, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:16.518748+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "3eb44f99-1764-46ae-8d59-848880dc674e", "result_sha256": "0bec985f129a2d1ee7d06969b7601f701d920ea3de1cad751781f3b38c91c85a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0114785, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 10.529710147995502, "model_name": "muse-spark-1.2", "output_tokens": 1957, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:40.898722+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "77f414e3-6edc-4df5-8675-0af2ece1e9a7", "result_sha256": "e5efc0814f156b82bda8dc4d0ad89c043bb3eba87547a049b2c814afdf7552ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01079425, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 12.039460727013648, "model_name": "muse-spark-1.2", "output_tokens": 1796, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:24.113577+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "6272790c-6110-4c3e-9bf9-79abebb50a88", "result_sha256": "5a93875771d0a97e42ce52c03ee59e56a3699563ee40b3c28ec8ca1ef0d87558", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0085545, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 6.75144525396172, "model_name": "muse-spark-1.2", "output_tokens": 1269, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:36.410561+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f80aecb0-a725-4b5a-940b-2ba42dccaf90", "result_sha256": "0e1b5c73c7fdb4cd20fd693cb15b1a3149149b1d369cf179f0b762ac6281844b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0092515, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 7.568004612927325, "model_name": "muse-spark-1.2", "output_tokens": 1433, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:36.213755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "13cf571c-96ca-403f-8d63-ebcf51f5b9b9", "result_sha256": "31fde5a5b20babd52139c8355faf7e05494725d1bb7bf08a2056a8d3820ec174", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00881375, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 6.720163014018908, "model_name": "muse-spark-1.2", "output_tokens": 1330, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:45.570990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b1329601-fcc6-46f0-8f1f-e22b8d893ab3", "result_sha256": "608dd4d6dd3b4f286b5ab0cdaae1f7ad53ffeae14e63c7bb733c2bab0268b59a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00770025, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 6.876269077067263, "model_name": "muse-spark-1.2", "output_tokens": 1068, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:05.531782+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2760ee71-03de-4d54-9bd0-2b1a172dfc73", "result_sha256": "e1e0767b7d90fa36d8471d7d66e17090dbbefcb64e66f0d006ab8b7dfc6e9cc7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010977, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 9.662924465024844, "model_name": "muse-spark-1.2", "output_tokens": 1839, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:34.582068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "54d8d572-d7c3-4c19-8986-c43680057cac", "result_sha256": "b6bebb420cef679451713987e15c4265f08e505bb929d496373b46acaf9a2c70", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01977025, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 22.096016245079227, "model_name": "muse-spark-1.2", "output_tokens": 3908, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:00.976733+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc017509-4737-42fd-ae43-c9923f32c210", "result_sha256": "c85ae0d58418b4cffd4392f9db1f0ef4103ce1b94ddb6a0f608780fa49c2c43c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01450875, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 15.108454669010825, "model_name": "muse-spark-1.2", "output_tokens": 2670, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:32.264489+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ff61193-964a-4342-ab58-72d74932f38d", "result_sha256": "eb6aa3fac2b3bd48937c681b4371f5c41b3694d40e253d74d4a30179a1bc9e38", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02181025, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 27.824937036028132, "model_name": "muse-spark-1.2", "output_tokens": 4388, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:36.880286+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "fdc6b6a8-4a20-418c-b1e2-b945de0f56fd", "result_sha256": "83db00036b776f26aeeb0010a48c4f04cabd6ccd1d8fc0525acbf0d07ce0e9a6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0222735, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 23.933071587933227, "model_name": "muse-spark-1.2", "output_tokens": 4497, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:44.537187+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "7a49194b-10f7-4207-96bd-a9fd7185a37a", "result_sha256": "ee17041b3f228b6bbcd885116345f1101a3631937db2453245a69d0197d906d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004163, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.4691303839208558, "model_name": "muse-spark-1.2", "output_tokens": 241, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:06.073624+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "cfdb2010-cbb9-4e6f-8413-6ad34e685d5c", "result_sha256": "5e0a232c216aee6f97425c76372c1a290b13cce563bd5cc0fe41e414ce357161", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003976, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.5655918620759621, "model_name": "muse-spark-1.2", "output_tokens": 197, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:13.014876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "41659a79-ce11-428d-8ecb-f9643d5272f7", "result_sha256": "3c01ff2fad02517b694b70fb5dbb704aeb801a5f5af4b1c13e88adf2fab18575", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038655, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.7197849299991503, "model_name": "muse-spark-1.2", "output_tokens": 171, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:03.743796+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "9006aaad-b5c5-4270-abf6-ac930f03e44c", "result_sha256": "ab06c8000abd2e894b855111ecfb7f7a3769a1c091ef0f1ec50bea31ece994ce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0035, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.0807970019523054, "model_name": "muse-spark-1.2", "output_tokens": 85, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:53.336523+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "4cc419a4-d15a-4bb7-a1b9-e19ef5b58a15", "result_sha256": "a21e87fba30e8c54801ccafcad1310cff2621c4a00bede34c0e2e2a23414e2c6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044265, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.227851976058446, "model_name": "muse-spark-1.2", "output_tokens": 303, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:33.687413+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "5ab07a06-ee04-4b67-a4f4-52411f7b1493", "result_sha256": "fb922ced5c8d35db1c84f84277df9b27bbeebf0e11c060df7681b36c09650e1b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040355, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.7472971200477332, "model_name": "muse-spark-1.2", "output_tokens": 211, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:45.610378+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a7e2d15-9210-4bd6-8edb-f3b5e3c5d5c7", "result_sha256": "d3d968b0ed50e3a3f0959449a8014ed322023aa83d3471fde79bf654138f4cd1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003857, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.384220514097251, "model_name": "muse-spark-1.2", "output_tokens": 169, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:52.990691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "7bbaf6ea-2fc9-470c-bf09-67a333cb9233", "result_sha256": "91ccee36cf4a895b4c0523b5fc853530817058b6265bcf7bf80a53d5d31e5fba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040015, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.7658077260712162, "model_name": "muse-spark-1.2", "output_tokens": 203, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:05.604479+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e944a642-8df2-4f04-bd24-9034da207964", "result_sha256": "2feb99d8c9abe492768118660ff9e3346b9b28a99daa7abcfce3baf7d8163df8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00403125, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.2952642209129408, "model_name": "muse-spark-1.2", "output_tokens": 210, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:14.095400+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e7eb7647-56af-4c3f-8524-0d7607a77f21", "result_sha256": "cc41fc521b3290005c99062c0b299bef7e6ff15490f89182beda99de270f4ed3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00419275, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.84347283502575, "model_name": "muse-spark-1.2", "output_tokens": 248, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:28.021292+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "234b4930-73a1-47e9-a7e8-18ec8d894948", "result_sha256": "82da40221d1f7bdbb5ffe044bce92246df7f0147883e6b487ed0b709677a7b36", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004146, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.7720898890402168, "model_name": "muse-spark-1.2", "output_tokens": 237, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:54.651866+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "454c77c3-8c34-4c6d-94b3-b8d9df5971da", "result_sha256": "a13767145d69cc93774f59059e10fd27b8daee469e502aedd4db69f98b5f81d9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00443075, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.5305191460065544, "model_name": "muse-spark-1.2", "output_tokens": 304, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:52.608771+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e5a60102-c1d9-4ada-81d9-586bfdeeacf8", "result_sha256": "8981fd4f9b0b82a7506bddb3d37fe384e8a7fe07195a96bd926e48d4b85c04ad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004669, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 2.235084339044988, "model_name": "muse-spark-1.2", "output_tokens": 353, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:56.536056+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d34a5c8-d72f-4dc8-b507-7740eeacc5c8", "result_sha256": "2e2fce0a90ec07f69124e076121cedd364ce8df930af89c9ed7bd3613041df72", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004941, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 2.7518417260143906, "model_name": "muse-spark-1.2", "output_tokens": 417, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:11.723545+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c9998155-0186-4c5d-9f91-5209bd2a6bb6", "result_sha256": "4963572080d4fa8e30a2090541090f5396623aea7bdf1202e8beca60fdb1c475", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006318, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.2516940699424595, "model_name": "muse-spark-1.2", "output_tokens": 741, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:08:47.717851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "64a9a54e-0da1-4827-bc54-59b4d60bcc81", "result_sha256": "5441ae575038c5162e0327563bd6d343db439e08f6073fc59295d1c98abb626f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006029, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.525265312055126, "model_name": "muse-spark-1.2", "output_tokens": 673, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:08.510401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "47fdad5b-14a6-4bdb-9ec3-b662e41e0607", "result_sha256": "ef88e13236e8cf6221821425487f290af68d149eeb2f1d2d3b80dade7e144488", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00503025, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.004271220066585, "model_name": "muse-spark-1.2", "output_tokens": 438, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:29.474470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b72ef7cb-7318-47b4-bcb6-6cd68afd897a", "result_sha256": "4f645c17fff05cd130af909a2da5476f548b3b9a5e3df2eb6ff76f255cd6bac8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00624575, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.894386596046388, "model_name": "muse-spark-1.2", "output_tokens": 724, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:24.351804+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "edf81fe0-b3c0-47fc-a1fe-94dad309d756", "result_sha256": "86615b4381ae772070db98f5e5944443af28ae688b6275a8d09ecf7b8fbcf51e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00578675, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.696202198974788, "model_name": "muse-spark-1.2", "output_tokens": 616, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:39.233017+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2ef54c41-10d4-490f-9c9c-ab21f9d39bef", "result_sha256": "801d94b4ee45797386608e445141c1128919735ed49ae160e02f3af872205111", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00620325, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.161859157960862, "model_name": "muse-spark-1.2", "output_tokens": 714, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:43.898200+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "ab17a4b3-e568-42c6-9307-a812e4e00c68", "result_sha256": "1671ff3b36b494d3bea837f776038b0e12ced5fce38857741dfaaba94d84db90", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041845, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 2.8020781310042366, "model_name": "muse-spark-1.2", "output_tokens": 239, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:59.848988+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "3ba6046a-0a09-4fa6-bab0-3e20aa3fec60", "result_sha256": "4d8a53d3f5355a589fad96d1f9b9293c1fa643645b85200b6ba5b849ecc8afce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044055, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 1.7928618530277163, "model_name": "muse-spark-1.2", "output_tokens": 291, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:07.641289+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "0fc7caf9-dbc9-4b7b-96f1-2ed091d97421", "result_sha256": "16b4083d9d505615130d71f36e5bbdbd38f99ffce6efb477253a8d56691c25aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044905, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 2.0136960180243477, "model_name": "muse-spark-1.2", "output_tokens": 311, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:29.466876+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "7215d3b2-fcf5-4aeb-a252-192195e81bdc", "result_sha256": "7322b9ad2fac22e7c077082a2535155289ea2dd28248fb6762f89e957a6be292", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00443525, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 1.935351073043421, "model_name": "muse-spark-1.2", "output_tokens": 298, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:18.437426+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "99446fa0-7d9c-4c39-af8c-78421c997cff", "result_sha256": "aa5f3f0f8f29ad7cd48d542b2b5079fd8194f66f7e4f8942b4d320f794e8d25e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01729075, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 19.083074892056175, "model_name": "muse-spark-1.2", "output_tokens": 3324, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:02.123943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "74582cfc-0640-4556-a106-601bcd7af6f5", "result_sha256": "2fccc7b31475bfdb2d80dd5c2a57936880f7e763cb3057fcc0f663840d98b4f5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00791525, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 6.787124107941054, "model_name": "muse-spark-1.2", "output_tokens": 1118, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:21.511062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b31ab253-e27e-45ee-979d-2ff87190ebee", "result_sha256": "7d281416e7432115ed3af846ac571de6391fc8b37b03b34cb9f0c412de0e8d90", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00730325, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 6.321386317024007, "model_name": "muse-spark-1.2", "output_tokens": 974, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:35.062170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "aad6ceef-b42b-46c5-9cd9-2f355956bafa", "result_sha256": "1ccb7ce8f74b9255b67a2142c0a825b464683bcc2c40eff9e2a3e437afd0f182", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0083445, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 8.266924450988881, "model_name": "muse-spark-1.2", "output_tokens": 1219, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:19.134058+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "3d832d6c-c1b1-4817-a8ff-8ab6123791da", "result_sha256": "8a3f012721b2ee32788e42d4cb684db20a5212be632bbbe1c8e72a7e0355812e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004596, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.7126339180395007, "model_name": "muse-spark-1.2", "output_tokens": 337, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:09.471354+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9c0ae70-3e43-4066-912e-1fb06f95e2b0", "result_sha256": "1d3a87182c0212722017d47bef7b032823ed4122b4eb9c91de510b48b29472a0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00415825, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.7730010619852692, "model_name": "muse-spark-1.2", "output_tokens": 234, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:21.938132+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d64518d1-bdfb-42e1-b28f-e322963d69db", "result_sha256": "c43f960d0dae16c2a0655acdb778c64d52582c1c383b18e325430ad2226d15fd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00460025, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.992594221024774, "model_name": "muse-spark-1.2", "output_tokens": 338, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:44.866395+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "abdea9ac-7c48-4b1d-ab7d-9d2c1436cf50", "result_sha256": "ad7841122c703d6fe80ff03aadff25d9fb7527fb56319fa6e5ba4e691dfb54dc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00407325, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.4419071458978578, "model_name": "muse-spark-1.2", "output_tokens": 214, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:31.282950+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "73db7a78-dade-4af4-b213-77fec9111a9f", "result_sha256": "e239a1ae587bb07be98bb2608589012074e629890449aad98cc9d7082bc2f5ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00440475, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.7003626050427556, "model_name": "muse-spark-1.2", "output_tokens": 292, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:48.474077+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b317be63-740c-4f5e-b98a-4bbc1141cb72", "result_sha256": "744071d8e8df29ef87660fe115548b58569d0dd962821eac825a429d48f1b999", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004069, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.5534410229884088, "model_name": "muse-spark-1.2", "output_tokens": 213, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:23.817157+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa3dbb5c-fa4d-4935-9346-a1076c7acded", "result_sha256": "d219d373d1ff46856182c7d2521aa98dc7cb5acdd08fa3ecaff13c83de6eaf32", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00416675, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.32004954200238, "model_name": "muse-spark-1.2", "output_tokens": 236, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:24.693129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "0af2bec9-ce7a-4953-aa93-4630f76368a0", "result_sha256": "23c882502fd3c0c7471e20a5f8f2fe2c0915b46d17fe12a9721c6bdbfe2fb48f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00470225, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.733842964982614, "model_name": "muse-spark-1.2", "output_tokens": 362, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:46.645861+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "cc5c74cc-b727-45bc-ba67-6f07cb35a9bb", "result_sha256": "114fa8495387d9af9a4f89892e4bdce3be76a706e59bd910d175440c474eb067", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00556125, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 2.897013955982402, "model_name": "muse-spark-1.2", "output_tokens": 560, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:26.608652+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "68ddaa19-b167-47f2-8447-23863e6db003", "result_sha256": "d27165cb64bdc0acd059b58585ba7018a4ff5e50d476ab5571f88cbb554f5afe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054975, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 3.2909865470137447, "model_name": "muse-spark-1.2", "output_tokens": 545, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:08.413444+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b65c47cb-36fc-4c43-ab4d-83b86c67c553", "result_sha256": "c7c061266fe4dc77dc0deb57522215995f4df7a3b6f001789d75ed93fe36febc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009977, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 8.842077783076093, "model_name": "muse-spark-1.2", "output_tokens": 1599, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:31.742680+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "9db14bfe-67a5-46dc-8a47-b6d8327f28fb", "result_sha256": "a07718e4f4cdd4e671f5d632afe24815fa106a85a7eefaf74c530f19eac2a024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00849375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 7.659704041900113, "model_name": "muse-spark-1.2", "output_tokens": 1250, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:46.615040+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "82ddb8d6-702f-410b-9b82-a93ed538fc51", "result_sha256": "56aa8a903b5eb4a2e514fa1a751ba1e9c6cd0d290f57ff7a703ba18472060fe1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0089995, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 6.704364825040102, "model_name": "muse-spark-1.2", "output_tokens": 1369, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:54.129591+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "71829c37-6963-49d7-bb79-c9f130d329f5", "result_sha256": "8d31d5c7abc7b6fb10edf53273cfe45df8a34fffb7f671da5468fcd9c0e66467", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0127565, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 10.690787823987193, "model_name": "muse-spark-1.2", "output_tokens": 2253, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:20.942917+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "5aa9ac83-df39-4833-b4b5-cb7e1efa898e", "result_sha256": "e39204204cd42be66c117e5ae9cd7b5010759a6ac512fb6cc37ad9ad91042db2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010079, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 15.655934456037357, "model_name": "muse-spark-1.2", "output_tokens": 1623, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:00.335299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e706fd5-b6e1-42b3-8c02-5d2614b847d3", "result_sha256": "b744d1674d468b3c45523b9faa1755cc49698b87df53d7af274ed9c8d83a282c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012782, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 12.715429529896937, "model_name": "muse-spark-1.2", "output_tokens": 2259, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:51.667341+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "51c04792-809b-4d19-9170-c2516158a2f2", "result_sha256": "bfc15dd969be08317cd65011ece1335b5e4148010d6447d03d84f2f976d78b5f", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.011507, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 10.642080577090383, "model_name": "muse-spark-1.2", "output_tokens": 1959, "prediction": { "choice": "C" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:11.338063+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "496d3151-e21c-4a36-83bd-fe66d88f1ffe", "result_sha256": "c44fe5d750e13a5b1599e3ec052ddec2a9a07c90201b00e56ce8a49ec367419b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01715525, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 17.428897729027085, "model_name": "muse-spark-1.2", "output_tokens": 3288, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:30.045129+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2ba644c6-d66c-4242-aa41-fef15f7601f3", "result_sha256": "c3459c84b59c276b373d7ba5fd6bed32a5b77e5cc8ce079a8f9727d709c601e6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01358525, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 13.174600254977122, "model_name": "muse-spark-1.2", "output_tokens": 2448, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:30.809553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "6f0c6d12-609d-4323-8749-e12949d67a7e", "result_sha256": "217be959569b30f47b635f8633d3f08301eded189e8bd88938f1ec1cf10a95ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01803925, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 20.90646203898359, "model_name": "muse-spark-1.2", "output_tokens": 3496, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:57.147930+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "9d2b4fe2-48d5-4cae-857f-0c3c902c3acc", "result_sha256": "c7ce54716122eb97224a02375b3ec155bd37d80fb75e51c58a53234760f68415", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00486375, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.945031356997788, "model_name": "muse-spark-1.2", "output_tokens": 400, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:57.569708+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d0ab118-6012-42e1-8b28-1f46a5d9957b", "result_sha256": "33db0efa90fae2f91daee444ed62d711519836530488ad08fbc4e0ea3b4de701", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004171, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.3058922209311277, "model_name": "muse-spark-1.2", "output_tokens": 237, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:02.952672+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "7509c437-39df-468b-b207-37902493e5d9", "result_sha256": "28b86f5769cf4554532bc791e6237c9d919d24b3574ea20ed162f7b8fcd7743a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00457475, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.379415131988935, "model_name": "muse-spark-1.2", "output_tokens": 332, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:24.616131+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c5ec94d-995f-4c70-af71-79801d92215c", "result_sha256": "25e25749ef5ca2a4b4ae6b99cc7e3ae68d25cd084042585d5ad17b3e14e8f6a6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041625, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.7418662779964507, "model_name": "muse-spark-1.2", "output_tokens": 235, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:56.281838+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "9f7ad5d1-1782-417a-8676-78368bb56f88", "result_sha256": "3998790c7ec26ab1248d0579717045acf54225d9520b832c485cbf06091dfb9c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005463, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.4108391830231994, "model_name": "muse-spark-1.2", "output_tokens": 541, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:50.638524+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "34438b61-f1ff-44c9-9be7-a9b07028a2c7", "result_sha256": "d62331135cfef073246509c7d3ff41e61a8f0b63cae7f7abfa5b6ebdb7c8ca96", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00430275, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.513616901007481, "model_name": "muse-spark-1.2", "output_tokens": 268, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:23.071824+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c5d82593-2e66-48a8-b0eb-27e7d86648be", "result_sha256": "5785a77dcc1d0913bc4d29a63071c173bcaafa610d97b39978b10a207dd82c14", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040095, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.169022548943758, "model_name": "muse-spark-1.2", "output_tokens": 199, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:28.402601+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f49a2a8a-2dcb-4528-a559-fff579d4310c", "result_sha256": "12dc9fb1fe9442d8254bd8498f70a43cbf6919f3bd5acf2ed119cad93e47c24b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00609625, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.8690432839794084, "model_name": "muse-spark-1.2", "output_tokens": 690, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:49.740323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "65b0ba15-bc76-494a-9ffe-836d7c103f5f", "result_sha256": "c9799bc5c65f47f3290edf43999d53ad629c02e8088029606c02b39ca65f0ba3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003882, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 12.5502386019798, "model_name": "muse-spark-1.2", "output_tokens": 169, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:22.364180+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "71b9b2eb-9dc1-46c6-8b1c-e93e9ad7f321", "result_sha256": "e5d24fc3773f1e87c813185c9d0827c496c4ad09b6f9a0a91f5d62c950da0819", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044175, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 5.431506837019697, "model_name": "muse-spark-1.2", "output_tokens": 295, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:18.735594+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "09ee20c5-3cfc-46db-a31a-338d5764e1c9", "result_sha256": "3b84da7437352a95cd02fa8495bf4c8a3feb7599d6d1f643b51a0939c9866558", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00386925, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.689811595948413, "model_name": "muse-spark-1.2", "output_tokens": 166, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:35.269954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "9d9c9d5d-fe90-4f69-989a-a45ea25ed2c1", "result_sha256": "517eec90e98ccca843c09b53a7d70b809f4a8da49b830115bfa4418e390f0211", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00482975, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.290991888032295, "model_name": "muse-spark-1.2", "output_tokens": 392, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:14.531564+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "84bb932a-9f9b-41d5-b5d7-5b38065520ff", "result_sha256": "3990eddf4b59a3de79e15f3c90ec04405671c98a19f3c9e0e4947eb19dedde52", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00427725, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.1054686550050974, "model_name": "muse-spark-1.2", "output_tokens": 262, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:48.106454+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e5971b98-bfa1-4347-b68b-7f5f4f4d3737", "result_sha256": "a6a1dbdc34fdfea490b5fb244daf12169f1498bb99046d18eb05ae5e11ef93d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004647, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.31621462397743, "model_name": "muse-spark-1.2", "output_tokens": 349, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:37.002380+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d223c1b1-8f9d-41e9-a9e9-9480ba813949", "result_sha256": "d2ddebba97493144014b7d0d4b3e57ecec1f0faa7708a85da2a687705cae1d13", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0042475, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.37120270205196, "model_name": "muse-spark-1.2", "output_tokens": 255, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:40.938707+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "bd8cf360-0bb2-4480-bbf1-242e4ada15e8", "result_sha256": "27ee8827a066432dac6ae74577d737ed7aee27dbdbd541f7897b823b6977fa3a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00446, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.3023848199518397, "model_name": "muse-spark-1.2", "output_tokens": 305, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:35.421200+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "01bf962b-abde-40d9-bf61-d569e183953d", "result_sha256": "44c7fe94d4ec244c42921ee70192e7fbdf62d17349d81dcb62c160f84250ac98", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0053865, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.799622353981249, "model_name": "muse-spark-1.2", "output_tokens": 523, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:16.338092+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "360a9821-2e34-4113-a387-2a94a6fce4f7", "result_sha256": "7c6ef71fb85d16d29289ddfe522c1490d52ff8b0857ca459ad7331dcdb53600a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00452375, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.848124483949505, "model_name": "muse-spark-1.2", "output_tokens": 320, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:18.161587+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b06aa486-ffb5-4fa2-927f-b15237c3d769", "result_sha256": "27ce423473abb4ed51a627300cd00e5cd2e0e5a225428f4d1ae97715404e0753", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00439625, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.4987557029817253, "model_name": "muse-spark-1.2", "output_tokens": 290, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:55.587887+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5746f6a-2d31-41df-a953-9af5c715082d", "result_sha256": "5eb7679ce9bdf0b3f88eb383edc4eb0e69aa6293a3468eb55961de6c66bb1b51", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00466825, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.522034148918465, "model_name": "muse-spark-1.2", "output_tokens": 354, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:48.892475+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "43a8a1c8-59d9-443f-8a75-4fa619b3b187", "result_sha256": "73ca07935c0547233d4c853fb8ea1969997307b9b2c7107c0f12925324e771c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00400525, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.132495478959754, "model_name": "muse-spark-1.2", "output_tokens": 198, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:30.599457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d465e382-138f-423a-a5fa-e6bc585e3d64", "result_sha256": "df7b4db6acf0a3df1a0e3d29ca205bd7a4229eba3f232857d58c4c0c026d83a4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0043835, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.8070053820265457, "model_name": "muse-spark-1.2", "output_tokens": 287, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:17:07.570693+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "48bb5e1c-58bf-4105-b15c-a2de2830f5db", "result_sha256": "bc33ae4b13040620caa2d8d3dbe309a3f467087098879d4f5738d52ada04f2f3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00418375, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.578616491984576, "model_name": "muse-spark-1.2", "output_tokens": 240, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:39.515774+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1315e1dd-881a-4915-b841-add609aee427", "result_sha256": "3185d1705c709c6fc4e43a813a350bc9bd7d6bcd079205030fc5a015845e17c0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00472775, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.264912256039679, "model_name": "muse-spark-1.2", "output_tokens": 368, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:15:08.429634+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3391425-772d-46bd-bcdb-bea2d94117a8", "result_sha256": "954e835978b3ffd770679ec254b1fc76ab76434e76c00fbeba5e63e1ee809ce1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00519925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 3.240561430924572, "model_name": "muse-spark-1.2", "output_tokens": 476, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:26.055121+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "edfd6836-191d-41f7-877e-a0568b832858", "result_sha256": "e53e45d69f1c92ebf330a9771133612ed560a5afaece2e76eb9d1b272295370c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00698425, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.764425178989768, "model_name": "muse-spark-1.2", "output_tokens": 896, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:35.446299+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "16d8672b-e882-4e1e-9887-58b68727f07e", "result_sha256": "5352e038dde61fd25203433542062e1c0aa285867b096081af58892daf89baa2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00544575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.147739412961528, "model_name": "muse-spark-1.2", "output_tokens": 534, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:19:33.281845+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "37318aeb-df51-4747-90a9-7a7594dda48a", "result_sha256": "93f5a6b198d52b3b3667502bb76fe3f68d96f7965f258ffb3463d212efb6c08f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005603, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.116557617904618, "model_name": "muse-spark-1.2", "output_tokens": 571, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:24.765696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b801c80d-3a79-44d6-838b-b9fadad3b843", "result_sha256": "ca08b7f0b5d06cdb085940d3a9a91a47d38adf2eb582fa88219606e2df976b54", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004991, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 2.4157997770234942, "model_name": "muse-spark-1.2", "output_tokens": 427, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:05.989735+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "cefa63a3-693e-4478-a371-a554ca88b73d", "result_sha256": "738600f156721dfd45d82e3190eff2373fbd8602071b9736e083ce52bdfcac3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0059345, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.0798594179796055, "model_name": "muse-spark-1.2", "output_tokens": 649, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:07.986542+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "7010272e-39a9-48ff-a83e-e57a15933454", "result_sha256": "723f2fcbfe823e70cd946672b5b1fe4b0c453ad2dc6c4f6d901851887bd4efbb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 2.8891125390073285, "model_name": "muse-spark-1.2", "output_tokens": 545, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:37.143095+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8b541b41-ffd2-4613-91ff-2d28b7beed1e", "result_sha256": "a78ee5b4275a775c7aa9c1b65b0e48cf9a3de5bbf2c373b6aa710d8dfa98e6c3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00808075, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.784529292024672, "model_name": "muse-spark-1.2", "output_tokens": 1154, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:30.805434+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0aff845-e3ba-43a4-adb4-747a77b88ab6", "result_sha256": "7660f45b19190d4919e77664a507600e58f3b32deaa0020d44518a883c5885e7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0062065, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.265611186972819, "model_name": "muse-spark-1.2", "output_tokens": 713, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:07.324353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "0b73f6c6-13f2-4876-a2be-2d7eac9b51bf", "result_sha256": "16ddea0e662c0ffae6f6d44df28d81fe00bbcc8213b9f1575a258ab26d6d41b0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00666125, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 3.8702842589700595, "model_name": "muse-spark-1.2", "output_tokens": 820, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:32.919518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c9ce7c6-09b4-4e2b-892e-a4fd64f3fd55", "result_sha256": "d28b182d574998d0ae7ce1a4b3d1396d23066e83d5799015158b476e9da26808", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00758775, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.458426292985678, "model_name": "muse-spark-1.2", "output_tokens": 1038, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:24.011089+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "ad07f7c8-c5e1-4428-a9dd-d341212e66f2", "result_sha256": "241692bc2c6e49af9ac84454babee8df6ff570de86a3b06439c70ceab98b997b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00505475, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 2.844955071923323, "model_name": "muse-spark-1.2", "output_tokens": 442, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:47.221940+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f42e1310-33eb-436e-aea7-7bb27e8dfb7d", "result_sha256": "aa7bf2dbef68ed334d6b5382354fb86c3e0e42a77484979b6b68ecdcf42a8d33", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00845475, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.04805161699187, "model_name": "muse-spark-1.2", "output_tokens": 1242, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:24.273224+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "3b6b6e7a-d348-4d1f-85ec-df981ca5a607", "result_sha256": "abf0a38bed1180ef37763ffb10025b8386da3168d350e5ff03fedab2bf303d79", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00604925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 3.762788377935067, "model_name": "muse-spark-1.2", "output_tokens": 676, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:23.574388+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf61abec-1591-4ee2-84f2-b17bea5acc9d", "result_sha256": "5cf5d17b673d6e6c9db088b84caa2e8cf14f1efb2b609707627c8a46aab6d9d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074645, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.3754498270573094, "model_name": "muse-spark-1.2", "output_tokens": 1009, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:41.749231+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e25ffd6f-4f4d-4aeb-9a85-6ddf3f15d7f4", "result_sha256": "972454ec907cdb1fb35de06e530c6f7a08b69bc656e26f0ae6d0d0065ba6bdcd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070565, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.397191503085196, "model_name": "muse-spark-1.2", "output_tokens": 913, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:12.828268+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1237b5f2-9478-4ee9-92ae-de29bcbd1fa5", "result_sha256": "426cb2081640867cfa997ff134123bfccfa3876faedea0ec5a644f2cf87df9ba", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00650825, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.82626821892336, "model_name": "muse-spark-1.2", "output_tokens": 784, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:18:28.221668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d943991-2dd7-43ce-b64e-eb17969c0557", "result_sha256": "acb2cfcfa671205f9c8a46bb8cb322cf12bcb78fc24e1d42d58710a1c57d4a21", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00663575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.198105251998641, "model_name": "muse-spark-1.2", "output_tokens": 814, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:35.464561+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "9beb2353-992e-4c86-a812-0ca44e860b01", "result_sha256": "0a8b3a05bf489134c971bfdbc5c57391409b60a749394043a87a031bbc13af5b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0060025, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.058040195028298, "model_name": "muse-spark-1.2", "output_tokens": 665, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:28.666934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e5241486-cdf0-486b-8064-0feb681c3685", "result_sha256": "04a1f3d746de149e9224df2319d1f4c7a2f4c413e3dd5c98ac4c66e34a7141b3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00808925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.940864754025824, "model_name": "muse-spark-1.2", "output_tokens": 1156, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:17.789846+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "fb820920-3c68-41b1-8476-98fc900e111d", "result_sha256": "477b1eedb52bb7fdd098feefdb645a973ca34b23b5db4df5c6a818055408b60a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005552, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 2.8945549589116126, "model_name": "muse-spark-1.2", "output_tokens": 559, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:50.063719+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "df800743-a7c2-4f28-9109-d5307db0b5de", "result_sha256": "da83e6b239ec32d1948ab38a16a7d4adb312057db7cf7a510306a0af21613f77", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0058495, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.30409041501116, "model_name": "muse-spark-1.2", "output_tokens": 629, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:03.313124+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "92f1a4b6-28c4-4671-8fa2-ed291142627a", "result_sha256": "56d6d26349f1d2cf5f95226c255a80598d97590e00bab60fca0a36a30044d492", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0057305, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.513560279039666, "model_name": "muse-spark-1.2", "output_tokens": 601, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:34.087067+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8406fa47-681a-4755-85fa-4ceef25b9b72", "result_sha256": "c506883e9296822e97ab62efc9aae3b6c626af4d733bfdc329b53ec2989abfd3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00587075, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 3.629867520998232, "model_name": "muse-spark-1.2", "output_tokens": 634, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:53.055914+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d5f81629-ba10-4e57-b237-e4e55dd864af", "result_sha256": "bb9d7d01b5028a65f1e718b1de78c6f76676d80477444099fbe409f68ab745ec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0093575, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 8.79694194206968, "model_name": "muse-spark-1.2", "output_tokens": 1450, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:07.339367+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9badc0e-1292-4051-a76c-38046dd7b451", "result_sha256": "a1abe66e9dd90f8d06ef0955fd1b18714d9dc133bb0a10100c6ec996be73944a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00839275, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 6.8556014619534835, "model_name": "muse-spark-1.2", "output_tokens": 1223, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:25.850377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "86158311-62a4-45c1-8fba-892bfc6ea6d7", "result_sha256": "b33815c0e7eb5ad3013fd806b18c52692bb4f627a76d34678333c20ef0902a51", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00891975, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 8.210171347018331, "model_name": "muse-spark-1.2", "output_tokens": 1347, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:20.093668+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0793ecc-6329-4f43-adc1-db5007c09a96", "result_sha256": "89255dd44baeca3eb1d47ce7507a5c628f11d3938959fec8bdef803dac465c92", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00839275, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 6.587025379994884, "model_name": "muse-spark-1.2", "output_tokens": 1223, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:53.737092+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d439d5e9-f13b-4e98-9760-3167f611626a", "result_sha256": "623c4db7ea25d49afb34cf664bcb1fde53f5fd522b98d981230dd409bee46926", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00705825, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 6.068606765940785, "model_name": "muse-spark-1.2", "output_tokens": 909, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:33.327114+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "a4735712-7188-449b-9e11-1f0fd5bdf3e7", "result_sha256": "d3adb55c2368e2ea92fee21e254b229e1345e9f0258a5565ede298f20047f60b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00657375, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 4.964360686019063, "model_name": "muse-spark-1.2", "output_tokens": 795, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:27:38.280195+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2340ef47-03fa-475b-aaee-9603649aa6ba", "result_sha256": "6633f1e2545d7bb73927eb5bcfb593bb7138fe2693386ea95655a3f23aba2892", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00760225, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 6.321251746965572, "model_name": "muse-spark-1.2", "output_tokens": 1037, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:20.364969+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed07c130-647e-4287-bd77-bd0bf7c39954", "result_sha256": "a97dc6aaca8568e7e7e93b530b2ff62a0e2b671f3288bb93550e57dbcbbf11c3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006187, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 5.9965426289709285, "model_name": "muse-spark-1.2", "output_tokens": 704, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:16:50.513667+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "be4ed3e9-c8e3-477c-b641-a6234966cd91", "result_sha256": "0659bd9d92e2a248f9bad150b2d9e583c52e3744d75a5fd765c89a1529a49b96", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.034696, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 42.38084592693485, "model_name": "muse-spark-1.2", "output_tokens": 7412, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:21.883411+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a92d8a0-07af-4258-8458-8c760b29baf8", "result_sha256": "4ccdf5f4a5debcb90d269f45f50ffb240c1b17aed669257546bf8cc67f671bd7", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.015469, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 14.688429589034058, "model_name": "muse-spark-1.2", "output_tokens": 2888, "prediction": { "choice": "B" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:02.408579+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "062a99de-b100-4d09-ab1e-81f35fc8401d", "result_sha256": "234d52979bfc19a3623424fabb4d36ef642d6aeee794f6e5d0aac1f4bc873a21", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02681225, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 30.552193701965734, "model_name": "muse-spark-1.2", "output_tokens": 5557, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:03.927180+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "ee4cb311-7b32-4043-a4ea-d8ca7be8baef", "result_sha256": "5c6aa7a5c6ce811cf991bbe485bc96373b296c00d205387ce7a9b12696898ab7", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0096465, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 8.441316016949713, "model_name": "muse-spark-1.2", "output_tokens": 1518, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:09.767986+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "911db5a1-0295-4dc2-b3f6-93479cdd502b", "result_sha256": "14df45d5d54fd41651ec3f2d275bd0b7a80cd056354dc9632f2811435ffd92df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00721925, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 4.883434049086645, "model_name": "muse-spark-1.2", "output_tokens": 946, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:25:56.083419+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8efd3a76-ec9f-4d91-9606-830c87800fc1", "result_sha256": "4eb16c94fadc235193de6f254ba49a516dd523197c0b3c680da65f70236f4220", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01872825, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 20.348303647944704, "model_name": "muse-spark-1.2", "output_tokens": 3654, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:25:08.807671+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "23c391b9-f734-4a41-9803-12afbccc2e9e", "result_sha256": "5500232f2d6bc37f3e96f89370319af26d04754b34c11bb1545af1f302aa9b83", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00975225, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.467217574943788, "model_name": "muse-spark-1.2", "output_tokens": 1542, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:12:55.397740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f17f62af-b950-427d-b70f-b02bdd24332e", "result_sha256": "f9a03bfb9545eb9877225f95a6e2ebec441b97702e034b54997cb4fc185fff47", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012264, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 9.5531799410237, "model_name": "muse-spark-1.2", "output_tokens": 2133, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:27:24.653611+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1d08ba4a-b886-4044-9f7f-e1972fb7b273", "result_sha256": "c53dc1c1337579dd4ff16798fcfd13679bd29f3f34531673ceb1bef036f83ea7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01002425, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 8.186191237997264, "model_name": "muse-spark-1.2", "output_tokens": 1606, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:59.534839+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "9ae5c4df-40f9-4a7a-91d4-f1db4b490532", "result_sha256": "5e5c265990b57f4080b5c390366461b8a9194c0a218f4570353562c54d9629c8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009918, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.615564880077727, "model_name": "muse-spark-1.2", "output_tokens": 1581, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:14:46.289096+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac1ffccc-f1a7-40c2-83b0-7cc04e7257a0", "result_sha256": "bb93a0c39e13730e5c19284cce42085e4a35bfe8f9dd8ac3455c4e3cc7c4cdb1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01553225, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 15.741402006009594, "model_name": "muse-spark-1.2", "output_tokens": 2902, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:31.154959+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "938875a3-c0f9-4e0d-af5f-e4eefe92913d", "result_sha256": "e36b07fd4498d0c2963dbe032492258dfd092d6ff444d49e97431e2e39d194be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0124935, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 10.73390313400887, "model_name": "muse-spark-1.2", "output_tokens": 2187, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:51.199670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "05700bc1-7c7d-4fbf-a41b-e9facfd0c2d4", "result_sha256": "498f0eea59636eb5b133d40f32f3e7edaecea429735666f02a662b5044298e00", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02550275, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 26.348972603096627, "model_name": "muse-spark-1.2", "output_tokens": 5248, "prediction": { "choice": "A" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:09:49.047799+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d7c3409-d323-455f-998b-baf147042630", "result_sha256": "31adb5727e72dfd4d80c54ccb797629c65302ed8496511845d61c75d0d98124a", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.007572, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 5.0052459600847214, "model_name": "muse-spark-1.2", "output_tokens": 1029, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:58.697342+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c31e4a0b-cf6f-414a-a899-34427773626d", "result_sha256": "9e80dfc674cef3d9d3bf1217c8c1e476b84cad44633daaf4f15ad78fbe24210e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01938275, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 20.699035033932887, "model_name": "muse-spark-1.2", "output_tokens": 3808, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:13:23.711501+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "3de77d8e-2379-4cc0-8c15-d58bcf7b4d93", "result_sha256": "416d02dd7149a050aca3b419314a72048fc02d554fa6c0002e634fb6e93bf8fe", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0088215, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.541983081027865, "model_name": "muse-spark-1.2", "output_tokens": 1323, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:27:36.975407+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f3903bee-cdff-4768-a64b-c64deed3b15b", "result_sha256": "dcb177a996460cb8ae5a6220d482f16b22e3fd55f3f81666030a045b458b8dd3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01719825, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 15.259894605958834, "model_name": "muse-spark-1.2", "output_tokens": 3294, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:21.088882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1da49ba3-635a-4a92-abc8-65962d70ce96", "result_sha256": "276370f5f8c3a52b52b5e6000fbe5fa5dc40290f5275a9c1791650dd4c781baa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00925925, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.420080962008797, "model_name": "muse-spark-1.2", "output_tokens": 1426, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:15.176081+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f28be4d4-d1a2-4fa2-a40d-a7be8ec810c3", "result_sha256": "90725bf42fd150e78a018c100205e743d062bcd080d378e4bf504172210483a5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01109525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 9.661268474068493, "model_name": "muse-spark-1.2", "output_tokens": 1858, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:48.078895+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "56144d12-8c78-44b0-afa4-59be7c9c4462", "result_sha256": "6e30fcc6e709bb0fbb25a5b34c133f6f5bf0fe753d26a7fc59a65e96646410fb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00952275, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.302882313961163, "model_name": "muse-spark-1.2", "output_tokens": 1488, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:13:06.268481+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "026579a3-7add-476f-90ca-4f2f13109b5b", "result_sha256": "fc08673efc6a0256ebe2266541b0ae71eb15c6892f0af28443d0112cbb8dd184", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00440875, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.1163951030466706, "model_name": "muse-spark-1.2", "output_tokens": 300, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:55.476156+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "48f28e47-7b6f-4577-b8ba-182993a3a74f", "result_sha256": "22f400f5b94913a55204d52bedac1e9801d0ea7f7d7ca2706b744132eb6eaec7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004991, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.4019865830196068, "model_name": "muse-spark-1.2", "output_tokens": 437, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:32.777194+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1d00c65c-1475-4817-9ef0-cfa7da17152e", "result_sha256": "95f035c942f5c05e6b5713776c7666b5267d9380bc2b46de127c5f988421e8ff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00617675, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 4.547867043991573, "model_name": "muse-spark-1.2", "output_tokens": 716, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:21.487051+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d2d56e6a-070e-484a-be6d-ea8cc6f40d5a", "result_sha256": "0a9c30333673a1f6d85644aec01ecc991bb10d43cf57658753ce075c4fd2feee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051695, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.2064100599382073, "model_name": "muse-spark-1.2", "output_tokens": 479, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:30.415949+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "9bf9bc86-79fb-4f9b-8dca-9c032c6b2bff", "result_sha256": "8fbe37542b677798f95d90d2d9289554708a62f028e134bc33acd9dd60e54e0c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00429825, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.0557417949894443, "model_name": "muse-spark-1.2", "output_tokens": 274, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:28.169404+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "243dec30-d3a5-431a-8ae0-666893b6bee5", "result_sha256": "f383e7244e7dcd3a2f85ea4c82cf4adc83dae1da1e473d50628f2ca387bae552", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00459575, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.1542402839986607, "model_name": "muse-spark-1.2", "output_tokens": 344, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:36.048506+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "bd1d7c9d-58f8-41c6-8643-c837eb8d556c", "result_sha256": "2f91c552043e0327717df4574b4e0d75a4abf4767462e4d1fa646f2c11f94a50", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00411125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.5287994410609826, "model_name": "muse-spark-1.2", "output_tokens": 230, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:11.291337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "434a86ee-db69-49c7-b052-7a84c77a4507", "result_sha256": "6c13e6b2bafb37da900ed33bd729a6a146bc0742e2730f9cabc984e3714e9f56", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00428125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.66416120599024, "model_name": "muse-spark-1.2", "output_tokens": 270, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:09:50.318936+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "00ba07c0-c4d0-4123-89b0-6be47ac3f2ec", "result_sha256": "ade178d222e9569e271b33d092ecf20033276a025a4d755eb5de62b128f589cc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0036395, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 1.0562436049804091, "model_name": "muse-spark-1.2", "output_tokens": 119, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:01.162349+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "ba5cf04c-05f3-4a2c-90da-8b25b1feb441", "result_sha256": "86161f57ae2bf846dba77825fa97c43be8798a99ec0b3706a5492bccb6e31032", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0045235, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.0965032660169527, "model_name": "muse-spark-1.2", "output_tokens": 327, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:11.415083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2488e053-d050-4f1b-a5cc-0c1e5521e29f", "result_sha256": "31ab6abcf7c82c0e3b31284e6569275211937937bf3fa4ae505a842c77fd0d0e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003903, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.14749312994536, "model_name": "muse-spark-1.2", "output_tokens": 181, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:49.669159+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f51fdd90-6367-46ba-9cbc-4620e7eedf08", "result_sha256": "5e23b54ba44aaf5a8324d7860fe72fdd05690c14bc1504dc2be48c9fc2868f45", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00439175, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.7992785869864747, "model_name": "muse-spark-1.2", "output_tokens": 296, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:23:01.421571+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "287b2c55-cd2e-45a1-9ac9-c3594343fcca", "result_sha256": "1a0640b33cc054acf8e4d770b56828eab6f22c8f608cc020322579c84e9e6a18", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00402275, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.297762028989382, "model_name": "muse-spark-1.2", "output_tokens": 203, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:21:50.102024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "5164a6fb-b9dd-415e-9350-78700ed139f6", "result_sha256": "0eb226299fa4c39cf2375ebd583383b8a896c37c0f63a6dbdace1cf8fae0c22a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00364875, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 0.9282357629854232, "model_name": "muse-spark-1.2", "output_tokens": 115, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:15.309832+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f88bed61-8d87-40f4-8f73-5f70ce655152", "result_sha256": "041ef5047bc0e968c51b91372950ed686a2d68c0c27a679037879a5c721a8668", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004027, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6848621719982475, "model_name": "muse-spark-1.2", "output_tokens": 204, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:06.997031+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "9ceb5779-4ca7-4ccc-9739-08579fa087a7", "result_sha256": "c31b979efde903041824383ae99fc6fe860f616d3ca0aac77fbff19648b3f67a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040355, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.5389962319750339, "model_name": "muse-spark-1.2", "output_tokens": 206, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:09.670666+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b33d0a67-53bf-4e2e-a4c8-8c67a8a03df1", "result_sha256": "2d020edf09f9d02b4a64add347b9230dd5247be515704460b75267228e55b8ca", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.578466056031175, "model_name": "muse-spark-1.2", "output_tokens": 180, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:22.483618+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "31eb0eab-c521-4cb1-84e8-0c0b3b4f1cff", "result_sha256": "979aa25b8fbaefe7a646ce7e1d03fb6a81fab37e07ad59b94196ee5e146b5502", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00396325, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.4941320500802249, "model_name": "muse-spark-1.2", "output_tokens": 189, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:13.821407+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "9e3a0ea6-7128-4282-a533-3f65e9aa5bc2", "result_sha256": "b4a96a1554f69f8510700a30c16f1ed6dc450b88736bc1e6eb78dda5ca698ecb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00467725, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.2796558931004256, "model_name": "muse-spark-1.2", "output_tokens": 357, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:44.170698+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "58b1c783-9f44-44ba-ad35-c1add27f39d6", "result_sha256": "afa1de4fb5027c1f6e24c4b6b57a7a3c30d38550410d13ed71043444965541ee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041035, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.5774850669549778, "model_name": "muse-spark-1.2", "output_tokens": 222, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:22:52.754012+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2f4dc2d0-df10-4b7c-9e3a-0e9c3e542dbb", "result_sha256": "b32098f3074b7cf5c39dcc2b8182c641b99a9d467cae0cf3bec4c74ba0f8bfb1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00396325, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.5218042549677193, "model_name": "muse-spark-1.2", "output_tokens": 189, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:26.787912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "7eb4626b-7163-474c-9d10-7a2bc72ab16d", "result_sha256": "72bf11922eba7177761d7a4d492b9ba63151151b80159a0448e0b1d0523a1bc5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00404825, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.2448150649433956, "model_name": "muse-spark-1.2", "output_tokens": 209, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:28.354987+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7731ea9-a243-40bb-8e31-4a9ca2fcfb58", "result_sha256": "97310784cbb322cdbf7202b37716fc8bbc07f350f7458bc94beb81a1ac53848b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004129, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.9000874490011483, "model_name": "muse-spark-1.2", "output_tokens": 228, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:53.176634+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "433e7636-531a-4bd5-acb4-76ad81a9de57", "result_sha256": "c9f0df18527c4e29b25b9fadeb41be431b61ab86d22ae053ffa611723e760aed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00419275, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.8391146209323779, "model_name": "muse-spark-1.2", "output_tokens": 243, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:39.302477+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "6dc68cf5-f16f-4025-8c6a-5feb1dd0d8d2", "result_sha256": "bcef534c0ebbf2d8eb18699ec899540f519232cdb15ad2c2f75e5bf59d5bc4ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041205, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.2397822729544714, "model_name": "muse-spark-1.2", "output_tokens": 226, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:11.492749+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "122d2f91-e2bd-4600-ad1e-4f0f8a851a48", "result_sha256": "afc5583c600d8c0e337cdfff8f53c5e581ad8578644d0f57c6137b9b3552b2a9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004163, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6414696569554508, "model_name": "muse-spark-1.2", "output_tokens": 236, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:38.076205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "4048847a-084f-4a06-94f1-b17e48aafe55", "result_sha256": "ed25772d1670035806d425af07788b8ddc69e83684bc601b81cdeb6854339ffc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00411625, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6815197770483792, "model_name": "muse-spark-1.2", "output_tokens": 225, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:09.250039+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8dad8a14-5a2e-4180-adfb-604a59097028", "result_sha256": "826f955f58a32f086d8b7003077e414d17fdd625ef863254150637837748bc7c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004044, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6659641630249098, "model_name": "muse-spark-1.2", "output_tokens": 208, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:20:20.132113+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8974b1c-5559-4376-92dc-47fedb16957b", "result_sha256": "760471699af13d3b6d17acd7a977560fce8c8cb3a561b56c227f4e6455beac58", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00436275, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.0873870269861072, "model_name": "muse-spark-1.2", "output_tokens": 283, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:27:20.592352+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1646591f-5856-40bb-9b77-58e357378dcc", "result_sha256": "59fd02ae33d1c8c14b4dbd3ed514e0c39bc4a05f6173d6955d5b67f98b1d7210", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00410775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6827544840052724, "model_name": "muse-spark-1.2", "output_tokens": 223, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:45.286401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "869a7236-9b2a-4a42-8a00-1cb70a786000", "result_sha256": "ab3ed4cc8d6371b7f80bca2f453a148de3072a309ee27c468a6b75ade7103434", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00430325, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.9265115570742637, "model_name": "muse-spark-1.2", "output_tokens": 269, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:59.161020+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac59c937-8cb7-444e-bde4-88f320fe52cf", "result_sha256": "eb99b5649b56663c8a26bb0180c6d9065b9a8784977344b4d8d479a04ea7c5e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00427775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.3678953329799697, "model_name": "muse-spark-1.2", "output_tokens": 263, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:17.881628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b23e16e5-9750-4bcb-ad20-4e197ef4c8f9", "result_sha256": "9099a6d30fd32714f6f6b52048b4dd04dec6dea76be1a092e5bc84f37efc2f1c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00455825, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.833311936003156, "model_name": "muse-spark-1.2", "output_tokens": 329, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:22.623438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "781d1052-b016-4d26-8c58-fd416c5cff12", "result_sha256": "1d6b5f06540bc0863e6455e436c959cd0d39d9b64ccb4c0c23ccebdee0252a6e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00401, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.4457778369542211, "model_name": "muse-spark-1.2", "output_tokens": 200, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:38.138486+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "dde1830c-18c8-4d6f-8f31-bc5ec3cf3af2", "result_sha256": "603d0afab5919c4e7037e7f7efea4090662fa1b2e3e41c654e51debf1a7a3058", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004027, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.6606194220948964, "model_name": "muse-spark-1.2", "output_tokens": 204, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:28.862024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "eca7e76e-cd53-44a8-a7ee-d3b5733c0868", "result_sha256": "7c24ec962c5ea7a5e9c50c504ed27cd257f87d5cf08d00a3bba1fbebefaaf608", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004384, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.9339128630235791, "model_name": "muse-spark-1.2", "output_tokens": 288, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:11:25.562355+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c0296adb-2f04-4005-bb8e-ffab02cbb29e", "result_sha256": "30f5b215d83d7a5bcbb254e1d49a99afa048ec0834d499ed69f326308fc2b8e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00414275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.590652244980447, "model_name": "muse-spark-1.2", "output_tokens": 233, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:37.818779+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c9ec430c-b679-4ac2-9fae-9d44a1535579", "result_sha256": "d1fff5815926eb3f28dabe302e6655fd387536750bfb18aa40f35c8f1439656f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004181, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 2.4464185580145568, "model_name": "muse-spark-1.2", "output_tokens": 242, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:45.002629+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c821fd24-152b-4bda-9565-fbf262f3dc36", "result_sha256": "6e36d98ffeccc8de8062e07d935ad4dbc8bd28ecfe3d2a011f0769a7e907dbb4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0038155, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.5707673049764708, "model_name": "muse-spark-1.2", "output_tokens": 156, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:02.949507+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f146d7b-8a77-4de2-b0f4-fd975e8ec08b", "result_sha256": "0e72cb377cde49b90dc8f633fef002f8f8b4a7f3d6f583330d922333b8258470", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003977, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.6392574709607288, "model_name": "muse-spark-1.2", "output_tokens": 194, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:06.276595+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8ebfcbd-45c2-464c-990f-e2f798e505d3", "result_sha256": "ee45b63fe05fcf687154990affea4bae15c451b676a8206238017180a1e9f461", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00380275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.4954862029990181, "model_name": "muse-spark-1.2", "output_tokens": 153, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:15.820737+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "35833890-c46c-4a3e-868c-3b1466d6acf8", "result_sha256": "08b94fdf69483eee6713325251962cf7cae7d5db00762176e0f42eade04efaa9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00412575, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 2.2771029549185187, "model_name": "muse-spark-1.2", "output_tokens": 229, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:23.429258+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc2a8710-040e-4341-8026-cf1b41ccffd9", "result_sha256": "547a2d29c463b4300c8171d723730f3c8ac7447af610b1fa305e715669e42563", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003858, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.4006804400123656, "model_name": "muse-spark-1.2", "output_tokens": 166, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:03.848533+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "405350cc-5f6e-470a-8f57-10d215b12b76", "result_sha256": "fcfd939851f0962da9abb152210d35716ddba64727a74ff419b5d281b10b4dfd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00394725, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.4965054800268263, "model_name": "muse-spark-1.2", "output_tokens": 187, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:11:19.567728+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "65fb9601-4b61-485b-ac66-1953037d6efb", "result_sha256": "b05e91a62c5eae5c34991abd7a6a73f763855fde54a20dc44aa1deb3c30a8196", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00819725, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 7.200914154062048, "model_name": "muse-spark-1.2", "output_tokens": 1187, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:06.444896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "14024fce-682e-4924-8dba-cf258e7bcd4b", "result_sha256": "904835ec99fe6742ad06b5509323db32ae3fe1a3916d7d2004dbd22acbf1f512", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0058895, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 4.599513876019046, "model_name": "muse-spark-1.2", "output_tokens": 644, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:40.220570+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "cbb47be8-6a67-4d2b-a232-059f9d441a8f", "result_sha256": "2cabd41170dfcd928e4c92cfacf27dda900698a4df475ed6408d9c909943fa27", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00739825, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 6.231243125046603, "model_name": "muse-spark-1.2", "output_tokens": 999, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:49.721016+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f08bd890-a201-4f22-a1d3-2d76d69a97e7", "result_sha256": "0cdc403c632019296af2ea7c805262099f2963273ae24951c10a8bbc5b50dafd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006918, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 8.089940823963843, "model_name": "muse-spark-1.2", "output_tokens": 886, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:22.940447+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b4e6053-e7e3-4eda-8136-e51d8a7facc3", "result_sha256": "2b0e0b5aed8bd66645dbc75a6f2b76a58530e1bad04518e845defe52044f41bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00535975, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 2.989831362036057, "model_name": "muse-spark-1.2", "output_tokens": 517, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:00.628514+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c03328c-c5cf-4d73-8159-f82a402a817c", "result_sha256": "8d77f34b26555c70d9582f08efbb4bb8e5fd3a3944242f035c1ad70c36988cd0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00571675, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.098198502091691, "model_name": "muse-spark-1.2", "output_tokens": 601, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:36.021503+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "b560d0c1-2ec4-4d3e-b107-ee3baa746365", "result_sha256": "f61718f6f8073885649ded001b9ee1ee40b4cb8c821abadea9669fad61c9949f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00554675, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.4089005099376664, "model_name": "muse-spark-1.2", "output_tokens": 561, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:02.186842+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "393a86b3-1300-417a-8bdf-56419025b116", "result_sha256": "ea37707003096e1f35e15d0717a6f31ff4679734e8a4ef459cb31c33c90cc038", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0045395, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.9804895650595427, "model_name": "muse-spark-1.2", "output_tokens": 324, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:38.850456+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "c783aabe-6a21-4495-8783-e11d12690806", "result_sha256": "cace49f07319eb35b58930ea388cc199264cb2b1069364e8bf1b52eb8ea3e8e8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005432, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.408427205053158, "model_name": "muse-spark-1.2", "output_tokens": 534, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:47.619184+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "77aa6fee-b32a-44d4-82c3-55fc6994f3cb", "result_sha256": "a445a69b3b49ff5746682abce7ccd5d6dc0dbd040049f0c742dd99bbbbc90f89", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00604825, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 4.13537210796494, "model_name": "muse-spark-1.2", "output_tokens": 679, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:12:45.526603+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "bbdca172-37b9-4458-a332-122d39386ad9", "result_sha256": "16da929759f214eeaa72af2c6f43a9d8a0ee7e8b5323aa72f85e15e73461809d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004888, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 2.4598877739626914, "model_name": "muse-spark-1.2", "output_tokens": 406, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:55.032519+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "20ceb509-4304-4667-bec5-ca777f9fec27", "result_sha256": "ce7d25e3372366f71555f2494e624d78d495600ff19c97caf7d64d535cf8b869", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00467125, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 2.7067400850355625, "model_name": "muse-spark-1.2", "output_tokens": 355, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:37.730234+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "70b5372a-7ef5-4ed8-879f-779bf74be0bb", "result_sha256": "a72729daf77c2d77735528d92e3c70590c7e3873716dff64f8c2dd2b1038d7ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006163, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.939325519022532, "model_name": "muse-spark-1.2", "output_tokens": 706, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:12.182556+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "241c4d77-861b-4e55-afeb-c7e90243b0f8", "result_sha256": "e446e0b50326f7c686e96a2f231b2d15654ca7e530c31221fe798b59e4a3a6e4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049305, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 2.9195223449496552, "model_name": "muse-spark-1.2", "output_tokens": 416, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:11:59.397515+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "25d2be43-83cb-4b85-8e1e-ad014f92b097", "result_sha256": "3c2218d3fe848e5913bd875fceca07c124e1cfe0967a653309bb6057f1729b89", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00532575, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.3866187180392444, "model_name": "muse-spark-1.2", "output_tokens": 509, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:45.818377+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "cca8c25c-ba3b-461c-bbc3-9935c889ddcb", "result_sha256": "880bdc4bef5d350b42237c6ed8f11ddba4d70a7d53efce458acfbc94f7678561", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00592075, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.2572774291038513, "model_name": "muse-spark-1.2", "output_tokens": 649, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:55.238857+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "90a70edc-009c-436f-be26-c50d87cf3204", "result_sha256": "39a220b5d24de56263f3077f3d2307dc80829fb6843609820cf422891fef2060", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003662, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 0.9933278320822865, "model_name": "muse-spark-1.2", "output_tokens": 124, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:06.862199+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f008dea6-15cd-4548-bf7d-8df98775f89a", "result_sha256": "de8413e41c42291f83bc4ce03ca7309c48d1780d960443bcf0b8cc12392c13d4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00404025, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.5757022839970887, "model_name": "muse-spark-1.2", "output_tokens": 213, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:33.903170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8a7c686e-d1b2-480b-bcd8-7c821039d5af", "result_sha256": "ceb5f61c4da265bab13fddb3c3f2e500654c4d66d4f0664e1751156cc7b18894", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00375975, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.0948480110382661, "model_name": "muse-spark-1.2", "output_tokens": 147, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:51.212148+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e087079-7a9c-4a4f-98e4-d09f485b2a51", "result_sha256": "605573b35aa89f6720f1bf3061366955867bd63e5521e06903407c8cd0b30fe9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00432925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 2.4141277230810374, "model_name": "muse-spark-1.2", "output_tokens": 281, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:26:13.040965+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "a163f291-78b7-4da7-aa53-5946677bb7c7", "result_sha256": "9c4a3f66b7ec7365d02dd2bbd024ec43ed8b4073142c3b10b846b6f1301031e5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00398925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 7.706908491090871, "model_name": "muse-spark-1.2", "output_tokens": 201, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:11.201441+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2cde75f6-42a9-4766-bf1a-7d8580aa9e48", "result_sha256": "8729489bf18e905c84b863cd89c6eb9061d5543ca27a19f6d6cefb6f64f2249d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004478, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 2.212465983000584, "model_name": "muse-spark-1.2", "output_tokens": 316, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:53.194934+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "281740d8-eca4-4fe1-aceb-c0e6774b95fe", "result_sha256": "f4f8cfc4b444fc896191d51774e8e3e9388c98378cd26f6a5aaab97d4f2985ff", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00375125, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.9841872620163485, "model_name": "muse-spark-1.2", "output_tokens": 145, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:20.275083+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "f8a7d7ad-1d42-429a-af9e-f57190312325", "result_sha256": "06ae0f42b6bd651bae3da840fc314c1fd8dcdc801052cef272e04696fc73c35e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00411675, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.762105548987165, "model_name": "muse-spark-1.2", "output_tokens": 231, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:07.995002+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c3c390b-55cc-4aec-87ea-4613afabc73c", "result_sha256": "64a04bbcf52dd1d05c7db84685122cb97940d4c140fb2332ee0be49ec1542ad4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00407425, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.1305083750048652, "model_name": "muse-spark-1.2", "output_tokens": 221, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:13:50.090522+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "099dceab-4067-4908-80f1-2c44f6b41fb4", "result_sha256": "b733eddfacb032b28b334b2f79894e86d5c5cdc7e16937f2d8b48800bf42d636", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040445, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.276593278045766, "model_name": "muse-spark-1.2", "output_tokens": 214, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:32.046429+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc3af050-0cd5-4d51-ae3f-8cc7a7e0434f", "result_sha256": "3b8a380360e42ef00e2a8e679341d6826850cff96f4fbb0beab39a0ab5028de9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0039255, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.4634585860185325, "model_name": "muse-spark-1.2", "output_tokens": 186, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:56.558276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.2", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c6b623b-4059-496e-ac89-dc4caf53e1f4", "result_sha256": "12e372ba9737a83f6cb1a2d851b657a4bee9800c0b24e22f93996f6f5ce8c920", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00392125, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 1.2428099979879335, "model_name": "muse-spark-1.2", "output_tokens": 185, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:45.156370+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "591cbb96-ff91-42e3-b390-32e2a63c0e03", "result_sha256": "febaf3296473aa77bba109403cba2d34fa0bf367d16628c295247bbd7d295d88", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004099, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.144107955042273, "model_name": "muse-spark-1.3", "output_tokens": 218, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:25:51.579227+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "7d7f786d-9f34-45f4-a863-bfacd594ec36", "result_sha256": "abfffb8076d12190b7a44d4d1b6ffbedc9138fcc0e63345a22d17db140c2d006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0045325, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 4.324630446033552, "model_name": "muse-spark-1.3", "output_tokens": 320, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:19.544167+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "563f7603-c09c-49f7-b33e-4e192561d0b3", "result_sha256": "e2a2de691740a8007f6c697b94c9b0026d3cadd7d10b5784f55ec5b7600da50a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00437525, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.328463578945957, "model_name": "muse-spark-1.3", "output_tokens": 283, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:11.552367+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "edbb2a9f-2ef0-455d-8050-81afe18babfe", "result_sha256": "f47b767dacc0e2b37e1423bef0f716f45b06e136a81e90fa7b4bc440e82728a6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0043285, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.8232133450219408, "model_name": "muse-spark-1.3", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:15.889684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c1a89fc-4361-4df3-b414-67de7eb44c70", "result_sha256": "1360403ddb6da47cd3f9f10b08edf17dda0e085e1af18db1bf6e0795c77d2092", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005918, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 8.185101227019913, "model_name": "muse-spark-1.3", "output_tokens": 646, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:19:46.851049+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "ec38d6b0-654d-4285-831e-b3f763a04924", "result_sha256": "1eeceda0138fb92cc3daba7081c0eebc39b52067a7f89b5dc28c9e7ecf7c2253", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0046685, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 4.698496990953572, "model_name": "muse-spark-1.3", "output_tokens": 352, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:59.099425+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "c576e86f-26e9-4cf2-a8e7-97f6a060f561", "result_sha256": "18ef988045709aa72ceed546a3932bc31a3d1fb098588dd17e7d4fa73128dedf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0053655, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 5.679681206005625, "model_name": "muse-spark-1.3", "output_tokens": 516, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:44.624037+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "55f1c4a3-5399-4640-959a-2f2139876f94", "result_sha256": "d9ecac3ac921b574106ff37675e24e58cef8eb0632299aa2249167680734b77c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00472375, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.9439842300489545, "model_name": "muse-spark-1.3", "output_tokens": 365, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:01.657430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "c2477d13-6c98-4c0a-8917-bf5793441701", "result_sha256": "313f0bd2cb3959e9b9518325525ff4dd7ed6dd699b8d86595c06fabf4d93d4f5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00467275, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.522048045997508, "model_name": "muse-spark-1.3", "output_tokens": 353, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:32.959076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "da61582c-df55-4c79-8581-d5cbece29c61", "result_sha256": "389e2988d840eeeecf11454ea4e274fb0aa5e3b3ec0eb3808e5c62aa6b6d55b5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00448575, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 4.049466694006696, "model_name": "muse-spark-1.3", "output_tokens": 309, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:56.365835+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "eeef3f86-6855-4785-b6be-275f739f6477", "result_sha256": "40224ff88e4dfedd11bd9bb370e862cda7f11ec066e1d6ab6b8aac02ca3cace0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00467275, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 3.728388553019613, "model_name": "muse-spark-1.3", "output_tokens": 353, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:33.340030+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "78aa0b9d-84be-439f-8f18-0ce2c13a0d91", "result_sha256": "2653ee894ad1ab4ffa1f29819a066a5ee37a947e558b7e0355c3bac87b3facae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051785, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2538, "key_errors": null, "latency_sec": 5.2100804060464725, "model_name": "muse-spark-1.3", "output_tokens": 472, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:56.462035+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "a3c74ce6-bfb5-498e-ac12-0dab052bc428", "result_sha256": "48564362822d1ec1ad5764f272517b0f9b4fabe605a4d23e640371ba0f63b9b3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004816, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 3.3717171429889277, "model_name": "muse-spark-1.3", "output_tokens": 367, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:58.809119+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "fa4f582f-2617-4862-9a13-0c5ee73823f3", "result_sha256": "1bf356bcfe280fa81070b18978d4865a3cfd5858b5f3b5cdf4e1a579133b6d68", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00574675, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 7.440627490985207, "model_name": "muse-spark-1.3", "output_tokens": 586, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:07.543740+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "733a5db0-7010-46c0-b464-bad75c0473bc", "result_sha256": "5d31d34cdd738fc58acd53e11b0e84f7282bd06613c12626ac640ff381d9916f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0050455, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.347017379943281, "model_name": "muse-spark-1.3", "output_tokens": 421, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:04.300489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5eaec37-d466-4018-af51-fefcbfcdf6b0", "result_sha256": "b2e771c6d9979d824e7f45bdc7b55d90f5307dc4b11229cbcc12ed9e309f5c13", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054025, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.423034635023214, "model_name": "muse-spark-1.3", "output_tokens": 505, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:09.554965+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "aa45d881-5cdc-4574-82e1-cf257e2e39f1", "result_sha256": "dd6287a4928c692b89fa6800b9302c45b226f4398549adae26428685dc0cc99c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00830525, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 10.706701286020689, "model_name": "muse-spark-1.3", "output_tokens": 1188, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:21.829970+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3e5c055-ea00-446c-be61-e1239033c7b8", "result_sha256": "e987436138a538dd9e3a0e91403c630da5aedd1ff10b05c4bf2a4ca0e17a3eb6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00480325, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 3.486556390998885, "model_name": "muse-spark-1.3", "output_tokens": 364, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:22:14.463275+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "e94b591c-1783-4f03-b59b-76e20a30c046", "result_sha256": "161b816bbc82cea2bc80e6eb25ef170b31761f54c1169429c70371e3f106005c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007094, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 8.926560633000918, "model_name": "muse-spark-1.3", "output_tokens": 903, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:18:51.634582+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d17f5fb-2a40-4022-a05b-5e21d0c98df4", "result_sha256": "e9f743870dd2c38aed99b64f851b6ad0a428fed626580e8fe26fdc2f3e1a2dec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049945, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.5344023630023, "model_name": "muse-spark-1.3", "output_tokens": 409, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:46.647213+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "3d8c5ca0-8672-4b68-9f42-d2aba14d226c", "result_sha256": "f8c927cf4684c9c3b443c848159b6ce1b150aaad18e6c5fba6861151088806a6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00515175, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 4.495390837895684, "model_name": "muse-spark-1.3", "output_tokens": 446, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:30.922179+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d5eec0b-513c-4a5d-a3c7-204bcbf1b5ef", "result_sha256": "b161a0235492d2d3767d3d8388a468fe790d29bb015b3060f9f4c57d8c9a0431", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006023, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 7.074936916003935, "model_name": "muse-spark-1.3", "output_tokens": 651, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:28.234998+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "03a4ba9d-167b-4531-a46b-32238d9aff21", "result_sha256": "8fc0457afe782e0930a7b296f962e56c0c842688b2543be0a6a2e450c83ed94b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004918, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 3.551690642023459, "model_name": "muse-spark-1.3", "output_tokens": 391, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:10.243742+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "92719f0d-6f7b-4dd8-a96e-6b3cebe74615", "result_sha256": "fba412654edc43961a6866c523cb087aa828c02429ffee9fcf5ef79e8b0b1866", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007247, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 9.19980038003996, "model_name": "muse-spark-1.3", "output_tokens": 939, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:41.728151+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "8fc764dc-fd48-4390-9405-bbd0b35acde0", "result_sha256": "3951aeda27b3f6602f251b5034c689c4a40bfb284df4bcf2799b913416a81526", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00569575, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 5.317758035031147, "model_name": "muse-spark-1.3", "output_tokens": 574, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:56.437033+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "37577f0a-822e-4a70-acf2-216a350b4b45", "result_sha256": "9777aa0fd12637aec73e912cbba7da1585a1a7209ffb2e5f7ee1feb83e274899", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00675825, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 9.42623075901065, "model_name": "muse-spark-1.3", "output_tokens": 824, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:11.448912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "a5f0dafd-8160-4961-9e80-85d7d91df470", "result_sha256": "92552177ffb03ce3563e3bf70e4a7a630ce60e443d9b099d9da59176cdbf6ed7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00544925, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 5.6755881229182705, "model_name": "muse-spark-1.3", "output_tokens": 516, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:15:02.524923+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "6bff5641-e152-4182-af46-20f0465e558f", "result_sha256": "50142e480f7e8b4d57b763fcfc62e7f1b0406af016e4459c489b78ba4bdaf01b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00745525, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2605, "key_errors": null, "latency_sec": 12.652785988058895, "model_name": "muse-spark-1.3", "output_tokens": 988, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:29.874773+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d1f46fae-98c5-464b-adaf-4c900d158144", "result_sha256": "b97e7abc7960b351b6becbf39a21eb0cad656d8d2bf3684df0a8f255e45273ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0042455, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 2.5206191450124606, "model_name": "muse-spark-1.3", "output_tokens": 261, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:41.831847+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "0bd06958-ddd7-43dc-b58b-b7504ce3d27b", "result_sha256": "3461903d14b207a8474caec443e2284b143cf73052e5b52572390ad41eabda06", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00456, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 3.319592054001987, "model_name": "muse-spark-1.3", "output_tokens": 335, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:13.935308+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "53575b8b-53d6-4b5f-afa7-38695cc5a445", "result_sha256": "8b1bbc68a619e3f9bdb6d86296735486b6e57a2bba5bafbc2279bb1a8d4679b1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00456425, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 3.4436037900159135, "model_name": "muse-spark-1.3", "output_tokens": 336, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:32.477879+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "75a8e47c-cb42-42f9-9c47-e378d4f589f3", "result_sha256": "85f1e4dea372bd5e3ef9afd46da9d6fe552796fbf3368e0ec6bfecb4beb0ff8c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00396925, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 3.6965691579971462, "model_name": "muse-spark-1.3", "output_tokens": 196, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:05.196985+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "a128121a-739a-47f9-b35f-85f77aa9551c", "result_sha256": "0d9d50e970ec82c073cf72c8ccebb20773a2297ebe5751d47c86aba4edbf1c61", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00374825, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.392521302914247, "model_name": "muse-spark-1.3", "output_tokens": 144, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:33.737673+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "eb68949c-5c38-47b1-beb6-bc45be9f22ec", "result_sha256": "8999aa9cfb23f408cb99787333b82123a66834dc83929f0869f67c5d614c5627", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040925, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 2.775418981909752, "model_name": "muse-spark-1.3", "output_tokens": 225, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:22:02.474627+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b44e7f0-bbd3-4c15-92a6-ce96962d1c18", "result_sha256": "2be0148750724455ac6e14a2c9681340d48ef0f0d8ce65adac246d2da8fc3e9f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0037015, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.3908638369757682, "model_name": "muse-spark-1.3", "output_tokens": 133, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:42.036161+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "352af212-5bb8-4dce-ba4f-35796b6c354b", "result_sha256": "064bbdc6d6b4a4818ade68242b21f84a77d9123584d91e4c73272f8e99983b25", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003948, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2509, "key_errors": null, "latency_sec": 1.9605802999576554, "model_name": "muse-spark-1.3", "output_tokens": 191, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:18.511323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f7ac12b8-885b-44a2-b3b4-dd5be45a7e1b", "result_sha256": "bc9e6cedad037f37e363bf109997f89788754a4c747db8f40bd211cdf9f55ee0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0073515, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 11.000559662934393, "model_name": "muse-spark-1.3", "output_tokens": 963, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:24:55.926368+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "075a7c4b-c4e9-44b5-a369-8958e0b920a3", "result_sha256": "0ecdb3093fe223baf1dde1ab14ea9d45066c5627715a536086b885a7f0f00798", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00738125, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 7.723235183977522, "model_name": "muse-spark-1.3", "output_tokens": 970, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:40.083666+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "dcc5b2c5-d70e-480f-9ef6-3ff4b3c7a249", "result_sha256": "34ac6c252e1a4f7551a191927dee9122d1551884f840d902e927b639479631a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00911525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 14.76292517199181, "model_name": "muse-spark-1.3", "output_tokens": 1378, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:10.580893+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "7b332cf5-69e2-46a0-8f10-1ae968859ec3", "result_sha256": "022fd1547ab65a1495567248be3bee426ec02f9ca54cb583c2b8af2cac527282", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0116525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 19.90948536200449, "model_name": "muse-spark-1.3", "output_tokens": 1975, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:14:15.651660+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d4fbce1-50b4-4f63-80e6-be767679d1ff", "result_sha256": "207a529e241ebb2d52b8351a5bcb2cb967fef7c5fb88561e06e0c9db5dbc9458", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0056855, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 5.02281672495883, "model_name": "muse-spark-1.3", "output_tokens": 571, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:12:20.933900+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "5ff78317-6326-45fd-b28f-50aa4a7873c4", "result_sha256": "4e21bb9b113f4f3aef7aa9703165f26eb4ef09f6916f1ab7ce0ba5de9dbee338", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0116355, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 20.44383939204272, "model_name": "muse-spark-1.3", "output_tokens": 1971, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:11:53.857315+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "efa7138c-0b8b-40a3-a08c-0f589e30bd01", "result_sha256": "5dfb6b04eea6da063d2349f509f66cf4ed513c5360f6ff382bcb86aab3516e93", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00566425, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 5.412234788062051, "model_name": "muse-spark-1.3", "output_tokens": 566, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:30.238047+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "35cd377c-c190-4d5d-b8d6-b3388d4f8936", "result_sha256": "dda44b41b0277c17a3e6854820010420df6a25fdab95aedcde392154f8aa2479", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008771, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 12.844864701968618, "model_name": "muse-spark-1.3", "output_tokens": 1297, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:19:29.034950+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "c3b96431-469a-4b34-8d3c-4aff4f6942c9", "result_sha256": "bb2cd7c18a433d34aa44058c3e9d43b7ee94e1803d32da296ad5d98cc042837d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007105, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.045810213079676, "model_name": "muse-spark-1.3", "output_tokens": 905, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:33.773381+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "46e80179-7e3b-4535-96cd-0f1fa84602d2", "result_sha256": "5d4e300d937bb90feaf886e13f5feb174871893d21d957a09036800bba334022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00726225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 7.896515092928894, "model_name": "muse-spark-1.3", "output_tokens": 942, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:15:30.952568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f3b82202-4066-49d5-b5c6-31c065461d46", "result_sha256": "1c8a00aba595a98bfb72ba6e0c41e687d0c8c90db81b3de2421bd1f5f1741bf5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00719425, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.3903076070128, "model_name": "muse-spark-1.3", "output_tokens": 926, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:59.485681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "499a41cd-a50b-4c23-82e8-1919e382f68a", "result_sha256": "f8f654b07da80c541416fdb02e8f8643e7aa3a89e65995d0482fc67996b19507", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008839, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 12.638847533031367, "model_name": "muse-spark-1.3", "output_tokens": 1313, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:26:06.884663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "dec815b6-86f5-4f83-92b1-de3c26addc8d", "result_sha256": "157badd0c77e28e57cdba8b06980bff795417202e991ebf05992092633b9eabf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00728775, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.588059064000845, "model_name": "muse-spark-1.3", "output_tokens": 948, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "openai", "raw_prediction": "{\"choice\":\"E\"}", "recorded_at": "2026-09-17T03:27:00.181396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "9257af6f-c82f-41b4-9242-a98d71925c62", "result_sha256": "03bab32cfb7e277b866ec39f6bfe39e3b5aa74cdffa74a198668dea5e12a6be8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00709225, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 7.600743173970841, "model_name": "muse-spark-1.3", "output_tokens": 902, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:55.152861+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4dff1fc4-f06b-459c-8093-60927c43dba1", "result_sha256": "ea0f8b55b7898343d0c8a616894b3628eab75d9991a552ab0214edde11f153ac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00668, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 6.337026598979719, "model_name": "muse-spark-1.3", "output_tokens": 805, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:42.170804+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d7e6843b-d4db-4e57-a81a-dc7c310315a9", "result_sha256": "bc54d8a4c02f927a289e6e9023e7bf5157ad2fbfa4f33c05519ab8feed3f179e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00792525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 10.42738997307606, "model_name": "muse-spark-1.3", "output_tokens": 1098, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:33.476546+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "fbe2a2af-eb22-40ba-8e81-3da879691a03", "result_sha256": "63c16166c752e21a89a720084697be5a1bee9448e46ba233872a9a1ebfaf1dea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077765, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 13.915445372927934, "model_name": "muse-spark-1.3", "output_tokens": 1063, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:23:54.018839+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "2d038b04-1f4a-4fab-802d-a8d7d7f18cef", "result_sha256": "1189fb702426332ed4ccc248367cee1cf3032008d9eb0932e088ac9e58afeeb8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01543075, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 31.501432609045878, "model_name": "muse-spark-1.3", "output_tokens": 2864, "prediction": { "choice": "B" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:17:56.072961+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "a40f6dd9-c77b-44c6-855a-fa3343676be0", "result_sha256": "60b2e8e06a200fdb5d5a4508af34e8720e4e42a942731ea12174f7ed30dfd41c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00576625, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 5.269284157897346, "model_name": "muse-spark-1.3", "output_tokens": 590, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "openai", "raw_prediction": "{\"choice\":\"D\"}", "recorded_at": "2026-09-17T03:17:07.233353+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e43f666-aace-47cf-9ab1-e718d553d723", "result_sha256": "beac17f69849db0bec0ab9eab9d2e6613df17cd332c2fe61ad2957c32a00fbb2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00942125, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 16.86148356902413, "model_name": "muse-spark-1.3", "output_tokens": 1450, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "openai", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-17T03:21:19.896700+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "38cb1ecf-8a5b-4f5b-98b9-45f203a313f5", "result_sha256": "adf2e1eb8b4940e0f34bbdaf7851656dc166fc29786426f5b26458ac1a2dcacb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01288075, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 26.9692703000037, "model_name": "muse-spark-1.3", "output_tokens": 2264, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:23.768918+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "25abd4fb-cdca-4876-8b47-7ea9633eac3c", "result_sha256": "3c2ce7e24e47ac5843cf1f1cee1f2fb6600bd5919a2167f879bc428f239698f1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0112275, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 19.424858235986903, "model_name": "muse-spark-1.3", "output_tokens": 1875, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:27.103321+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "51bf1997-f200-47b2-9c98-ee09bc616bf9", "result_sha256": "3b2b9c7ce896c247bf437b5771bf22ae84e87c9e88b904f7a6ea20f7de6e079b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01175875, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 23.651289117056876, "model_name": "muse-spark-1.3", "output_tokens": 2000, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:57.108407+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "51809868-df0f-4d83-a3cf-ab45ec2b1895", "result_sha256": "fe6c0f811f0b2c21e810d8c483a5e5797f4a63e76f562e3678f39b8f8d3cf46e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01868625, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2607, "key_errors": null, "latency_sec": 37.17285589396488, "model_name": "muse-spark-1.3", "output_tokens": 3630, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:31.732676+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "228e0333-71f1-4b21-a58a-0b095c433215", "result_sha256": "b4b5fbcb3450d1d9db1276b293799b772fe11abd7a3f7f8251431444380f2e5a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00508225, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 4.341189622064121, "model_name": "muse-spark-1.3", "output_tokens": 452, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:14.204517+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "ca2937fe-dabf-41d6-9ff5-3960fcf83cd9", "result_sha256": "0570b2687d9a341935ef046044f7c2e3646bae4b16f1d06464d854112ee137d5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00604275, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 7.286079034907743, "model_name": "muse-spark-1.3", "output_tokens": 678, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:25:00.053420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5f0e5df-502c-4a80-bc03-cfa1061e3280", "result_sha256": "37d72c052fb36f323c2928eae958691d559eb1ad089bdf344f91a167beade67e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0060555, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 9.509411437087692, "model_name": "muse-spark-1.3", "output_tokens": 681, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:36.623360+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "af504d29-b5fb-4af4-b594-a19220f127de", "result_sha256": "6006028acce039f9aba45b5813f78eb8e7d99b6c7c32cda69f33ba0090925d66", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00661225, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 9.482423731009476, "model_name": "muse-spark-1.3", "output_tokens": 812, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:01.883142+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "0eac881c-0f98-41be-a984-28d32c575856", "result_sha256": "2dc24e5cfc6143e1ffe593415a83e22275098e133f29d304323cadb3dcba2485", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004585, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 3.31866834894754, "model_name": "muse-spark-1.3", "output_tokens": 335, "prediction": { "choice": "A" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:03.747755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0e42365-c607-4785-89cc-236900832e2e", "result_sha256": "378fad1b97b21c1fe90f5ea9507829dec3c908d615c2b866fd8feab9c6ffb514", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00488675, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 4.636287408997305, "model_name": "muse-spark-1.3", "output_tokens": 406, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:04.769554+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f36c9f8f-1a4e-4495-8ab4-46245f268b6f", "result_sha256": "7f98fe86760a5889f0be23b302707f3d8b7fc344f15c71b8605b41e5d1ba5566", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00489525, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 3.254211710067466, "model_name": "muse-spark-1.3", "output_tokens": 408, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:21.784105+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "da10e6b1-0bc9-4849-8c9d-29d7cbfa6c52", "result_sha256": "56571a6335892ff166ca3d56eb04591b507621fa39a6e29c472e89d83f1e6522", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0049335, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 4.074470401974395, "model_name": "muse-spark-1.3", "output_tokens": 417, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:39.305380+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4e93f6a0-3743-4dc7-b567-b72716888126", "result_sha256": "68868a474a36f4742b61a61e427b69108ade47362a3ff2f1c5e142bd8bfdfe1f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0136205, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 29.440734864911065, "model_name": "muse-spark-1.3", "output_tokens": 2461, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:36.073376+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "e70e7e95-5400-40eb-b815-95f3bd112275", "result_sha256": "b345037fe76c763a8311d8e1555a623f064961de3f100f7b1592aaa39ed245eb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01298725, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 27.388376454007812, "model_name": "muse-spark-1.3", "output_tokens": 2312, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:01.018153+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "59d64f53-a7ab-4494-8e20-208064d17970", "result_sha256": "f48de8ddf8f2e18287f97d8812dfdcb7c98d9b1150267969c9303c7142707dcc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0127195, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 25.65650514292065, "model_name": "muse-spark-1.3", "output_tokens": 2249, "prediction": { "choice": "B" }, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:21:44.251743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "792c458e-cb57-4530-adeb-d6a0d150a695", "result_sha256": "0a2a0fc75a753cc2df3c7befb1ad04e3ba60c0337850a174ab2d1ac64d627263", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008937, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 17.539160526008345, "model_name": "muse-spark-1.3", "output_tokens": 1359, "prediction": { "choice": "C" }, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:14.906361+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "1bdf6519-856a-4b74-a27a-97655f418bb9", "result_sha256": "5181c78a01defcf93cab82f142410ccbfb9c42ac07a968d5a7231838f04b7f4a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040185, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.816687901969999, "model_name": "muse-spark-1.3", "output_tokens": 207, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:10.526927+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea9b27ae-3631-4aed-a3b3-df501d1d7ffb", "result_sha256": "7721e015b223f2e1e935d53c1f895512b14f9ff61477137eb94d6c55a110c409", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00395475, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.7662125570932403, "model_name": "muse-spark-1.3", "output_tokens": 192, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:23:25.338455+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "2e924907-b1ed-485e-82f0-c1eaaef92849", "result_sha256": "7b8219dd292e2779d6532775dcc693430e3a6befaeaea02740267aaa1aac9e4e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004265, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.673811913933605, "model_name": "muse-spark-1.3", "output_tokens": 265, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:12.938429+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f30b2240-a18d-4411-a42b-bb3ecc71ebf6", "result_sha256": "edf16df9d738e73aff9d81be7fb54dc8ab0bbaf3649791eb4f937e0a68d3bd6c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004418, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.577896742033772, "model_name": "muse-spark-1.3", "output_tokens": 301, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:44.751831+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "60ca5067-e328-4526-89cd-6808d4d83e02", "result_sha256": "b342ee0ef7c4ff4a12ef7b4aa586a8add4c797b5b7682df88c2c6e872b49d1c4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0045625, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.3197812070138752, "model_name": "muse-spark-1.3", "output_tokens": 335, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:50.116979+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c0b40c6-8b85-4c29-889b-c293dc5bc4e5", "result_sha256": "fe736e1ed95fbcd64031c184e4a0e89e5356d0835ca85ffec6ded0fefac4551a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00430325, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.031874821987003, "model_name": "muse-spark-1.3", "output_tokens": 274, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:17.221681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed0ba4d6-19cc-41ca-a420-babf59e6c656", "result_sha256": "2ac04e588b511a98a73acfd0bbd71e8c9f984d762374f4bae4aee955b1abad73", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004503, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.327036952949129, "model_name": "muse-spark-1.3", "output_tokens": 321, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:57.958744+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "c10bd7c8-e0d8-4987-9b63-079b246f2913", "result_sha256": "d0e0ee7a312943fde209794c1dd7386d51de9cda8a281ba5d6b884dc08a11011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00418425, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.6001617120346054, "model_name": "muse-spark-1.3", "output_tokens": 246, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:01.490886+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "6daebce5-1472-469a-baca-2c71706e17c9", "result_sha256": "07868973e6d29b59bf1f01af64a9b6c783f6f21fa7dcdcbbcd999b6a1f923e6b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004792, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.879870869917795, "model_name": "muse-spark-1.3", "output_tokens": 389, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:41.515596+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "e3599079-50b0-49ad-a08e-ee1771e004b5", "result_sha256": "d7750639e6ec25c9cd4e4f13217dfd2025699fcd0a199f833a24ffc03211fd68", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00477075, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.778705027070828, "model_name": "muse-spark-1.3", "output_tokens": 384, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:31.594696+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "9f1abcff-3e1a-437f-a78e-cd38d8660318", "result_sha256": "751feb62ab4d0aea668a453ff54d859efe4e06c252c9a1b73c7779d6a09f4464", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004333, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.832102475920692, "model_name": "muse-spark-1.3", "output_tokens": 281, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:35.705106+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f2a4dbab-4df3-4128-93ba-485286ca00a6", "result_sha256": "c849112182d9f0d1060986e6ab7f688a2d38c89c9e0c4bf0644bfd8e5a3f53a6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044435, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.7087591529125348, "model_name": "muse-spark-1.3", "output_tokens": 307, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:25.521689+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "ef231dad-e419-4838-9264-bfe8a3e6d0a8", "result_sha256": "ea4e8fe095c30e4b0bdb43ba2d37962f61b54fca5cf86b93013306a3dda89438", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00756425, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 12.641721123014577, "model_name": "muse-spark-1.3", "output_tokens": 1036, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:15.836460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "6a8a6ea7-0693-4830-8574-5142b123cc97", "result_sha256": "2d6ca7315fbbf3899fc41b67ebc510fda812d8638b55d89fb03d15bd1b49ab82", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075855, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 10.20547621708829, "model_name": "muse-spark-1.3", "output_tokens": 1041, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:40.388024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "62c8e6e8-6f68-4b2d-9b18-312c7ef72237", "result_sha256": "68256fa3dd9e17fe75d5e02ad1113d5f36faea0ff22bc180685410dab5e1f1da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00773, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 10.995992118027061, "model_name": "muse-spark-1.3", "output_tokens": 1075, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:22.984181+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "fa5d090c-ccbb-4e74-bc5e-59e78f03d447", "result_sha256": "225f477fb0c238da3049cac1c1084f56ffbf97f3dbb43a15582818090b1ed1e0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00817625, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 15.296553215011954, "model_name": "muse-spark-1.3", "output_tokens": 1180, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:43.997525+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "2672105c-bd9a-4264-bb7f-997709424325", "result_sha256": "a4961e1a5ca68910e7bf2955841ee19986c42579e840389a6fa9327ef1d6bde9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00970625, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 15.617339341086335, "model_name": "muse-spark-1.3", "output_tokens": 1540, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:43.930386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "33b82176-c09a-45d0-9561-2c4596fe2b19", "result_sha256": "439f882dad5104bd34a646b1177734fd0c42a707514560be61ede4d2b913b971", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006183, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 7.8986429730430245, "model_name": "muse-spark-1.3", "output_tokens": 711, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:46.749018+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "36ab0351-704f-420d-9a98-27c64bd84a71", "result_sha256": "9bf5c5da1d62d80ebd618da0d06f648197570e769f3cbbf073aa15a4b2dbcd1a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008699, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 13.931639526970685, "model_name": "muse-spark-1.3", "output_tokens": 1303, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:12.099324+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "804b1994-061e-4fd8-8ff1-13936d19f222", "result_sha256": "37f1caf6746e7fb82ca36b1ff356d76f45d20c2b0f6aaf5d9a8f366330de1c02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006387, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 7.837557581020519, "model_name": "muse-spark-1.3", "output_tokens": 759, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:32.813173+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d45e4555-13f6-498c-906e-c66e0331904e", "result_sha256": "21be072c55a8478b4a33683f9aa9c70e1d24e3e22cfc1d9480654ce7572177e0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0153375, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 35.40869758196641, "model_name": "muse-spark-1.3", "output_tokens": 2865, "prediction": { "choice": "A" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:13.756755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "eda522ec-8232-4166-a1d6-56b589ae22b6", "result_sha256": "53390572cfdd3c2dc47f935f6bef60314c0c7ff32601ee0395d459502c72306e", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.008087, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 11.726426254026592, "model_name": "muse-spark-1.3", "output_tokens": 1159, "prediction": { "choice": "C" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:15:28.357908+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "8c39acdb-c8db-40c7-a321-2f7c25722aba", "result_sha256": "74d680579703a7a016e0b9c36b2aa87b4266f7405a12db0885978e84e48686cb", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01208625, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 23.29569342499599, "model_name": "muse-spark-1.3", "output_tokens": 2100, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:31.937511+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "8cf0bd6b-15a0-4c90-a728-9994a80acc7c", "result_sha256": "1d1c5f1b472499d748ac293ccb26243205f5c3bbfd6ebe0f44e5aadf720dbb82", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00962975, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2529, "key_errors": null, "latency_sec": 14.277560935006477, "model_name": "muse-spark-1.3", "output_tokens": 1522, "prediction": { "choice": "C" }, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:34.784529+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "9853a0ae-30cf-4590-8c85-788709eebc85", "result_sha256": "3f1262b6e2212ed19baf1a54e77c85aada853e673c61d6a11cb976c056b1bee6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004248, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.5761458890046924, "model_name": "muse-spark-1.3", "output_tokens": 261, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:06.981620+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "89836f07-50dc-41ec-bfb1-85080e546363", "result_sha256": "ce060cb0c17bb6cd2e693d96438081cc512eb95e87787596636a66cd119a8857", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040015, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.113903548917733, "model_name": "muse-spark-1.3", "output_tokens": 203, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:13.277144+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "58bb66bf-aba4-47a0-8234-960a8a9be253", "result_sha256": "8f135bfa39688dec36c096aea43a49d005708b8bcbe0c1ddd28cb5893d8acbc6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00411625, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.2714380259858444, "model_name": "muse-spark-1.3", "output_tokens": 230, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:04.221536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "b563c5bd-45ca-4f3b-baaa-fe7e04f183ce", "result_sha256": "3dc143f7c1a870f5edafce741134bf26a60523065c0ad9176bfb017bd9188750", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00449875, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.7361105909803882, "model_name": "muse-spark-1.3", "output_tokens": 320, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:55.989557+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "2bd6e3ed-65e5-4b65-ae0e-01ba60a7f7aa", "result_sha256": "b93cf19f72478d2feb99a369db19f174afb3b0623cd20725b721cf167fb82916", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00406525, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.6619438549969345, "model_name": "muse-spark-1.3", "output_tokens": 218, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:32.930141+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "39466e6c-3de9-4b66-9dd3-f6422d8499e2", "result_sha256": "640b7b35014694dd7b989c0f2362de0f18e2cff374465191a316daee0aebbb40", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00462625, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.2013260659296066, "model_name": "muse-spark-1.3", "output_tokens": 350, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:46.565890+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "01b4e958-818c-4451-9678-90c65663a39b", "result_sha256": "2807348ccef92853c64c30aef0d799940b36b154ea62615e21762e561913a1aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00462625, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.7478385599097237, "model_name": "muse-spark-1.3", "output_tokens": 350, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:08:54.616274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f721d712-756d-4785-8190-d85d1b04a87c", "result_sha256": "8aaa77c9da749bc80306085f3676190dbfe2aa5e265950c8371fef5e5e20a0e9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00421825, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.3901778259314597, "model_name": "muse-spark-1.3", "output_tokens": 254, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:15:06.164438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "8830bdff-5658-435a-8e99-532e09aaea61", "result_sha256": "a2b6c4d6f936e48b4c6297c23cf1ba389711a46e07e5fb6c6d987469abe69fb0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00453275, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 3.540435035014525, "model_name": "muse-spark-1.3", "output_tokens": 328, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:16.223140+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd1b43f1-76d0-4b29-a2ae-c40fe798c497", "result_sha256": "a6c7c467bfb3dfa23f0b8ef75df3ffefd3e8954f5bd03f2ee581afe59235cbd2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00451575, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.6159100950462744, "model_name": "muse-spark-1.3", "output_tokens": 324, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:28.680567+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "57b43732-e96b-4a13-bfdf-61ce1f8be619", "result_sha256": "f3e798e55973866db9d5e7a1bfc1acc52e2b1c11ccef1dcf77e5cbab8399b954", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00381025, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 1.8686451449757442, "model_name": "muse-spark-1.3", "output_tokens": 158, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:54.735139+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d08a9881-1f9c-4301-99f4-54bb18edc55d", "result_sha256": "88ab6dc0c5050a3f23c846688e392819b9486aee80d4eb46396a7e26cc11fb44", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00438825, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2511, "key_errors": null, "latency_sec": 2.653142602997832, "model_name": "muse-spark-1.3", "output_tokens": 294, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:53.683388+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "7423952d-f87d-4368-9307-da47fb265ee3", "result_sha256": "7b051293a26591b6ac7baee30b8be1d9275fc35971688d57f350ef80d2c9392c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0050855, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.643806732958183, "model_name": "muse-spark-1.3", "output_tokens": 451, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:58.631744+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e2ee58d-e5bc-4b8d-9ff5-996717485e3f", "result_sha256": "ce553de01b6920d529415d53bb8d3157dfe55ac0aa5ffaee6707aa8552d0e06d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00603325, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 7.416966984979808, "model_name": "muse-spark-1.3", "output_tokens": 674, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:16.433655+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0c11b6f-2f05-4a42-9796-b1f9c9b3dbe6", "result_sha256": "e0927fc3257a3fb7fd5ecd554265ad42149577a3cd4dff9776f0fc9a03bbf844", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004992, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 5.405047936015762, "model_name": "muse-spark-1.3", "output_tokens": 429, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:08:48.544154+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "31b63c80-53cd-4564-8f07-aa749af7c7e8", "result_sha256": "c4d512b5ae6e3c9d1c2d066bde1ed7a1865af559dcdd2d104082b0eb5d25c537", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004856, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.192412376985885, "model_name": "muse-spark-1.3", "output_tokens": 397, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:08.011045+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "83520d0f-4853-458f-9a16-d3d39de3c9c4", "result_sha256": "3d0eab94cdcbc4aecdf688c87313c45507b98e065fd865c21bef233e5c069856", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0050005, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 4.377719170995988, "model_name": "muse-spark-1.3", "output_tokens": 431, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:29.745733+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "525b5003-a8e9-4b8d-8975-ee13977b3efe", "result_sha256": "d47f1aa471abab9a956ed52f002343e0b683cffa9ff195f408df95869a03181b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00516625, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 5.424577387049794, "model_name": "muse-spark-1.3", "output_tokens": 470, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:24.063943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f0c7cad8-7d0d-407b-bf49-c8a9cdec09cb", "result_sha256": "e2d8869b4d1e767e1a839c104e1b7ca971efecb07930be383ad2632a22813f0b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0057145, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 9.578903277986683, "model_name": "muse-spark-1.3", "output_tokens": 599, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:45.101068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "00af77d5-ed51-42cd-a289-d86831f84c38", "result_sha256": "4c407313fa9812270ae5d20013aceefaf1738559d2fc013f8031cf31b70e8372", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00469025, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.4732385240495205, "model_name": "muse-spark-1.3", "output_tokens": 358, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:43.114938+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "170bdd2b-164c-40cb-9580-6007a855547a", "result_sha256": "f2815828d84d6b7b9baa4a47200215d72ee20d324a4a0bd58262d033b46a4fb4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004635, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.281325545045547, "model_name": "muse-spark-1.3", "output_tokens": 345, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:00.138630+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "867d14a5-37c6-4a17-9297-082f3fd627ff", "result_sha256": "7786f4b4c08b6b8104d9cb3d2699fd38026858844fa187de746afc55e86eab0a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004873, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.6025799569906667, "model_name": "muse-spark-1.3", "output_tokens": 401, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:09.023124+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "5de45a43-4a35-4eb7-b580-634a2fd0fcbc", "result_sha256": "4fc72b2c160e46c2d18c647095e07a04457a37f767db1dc0652789e8681664ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004601, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.0402124479878694, "model_name": "muse-spark-1.3", "output_tokens": 337, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:30.303094+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "fd772bb2-e7c8-4f6f-9a99-047a21eec2c2", "result_sha256": "de80590a7afaaa907c2f54f3a22961d67a5904cc8d7d00449335327b98dd24df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004686, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2535, "key_errors": null, "latency_sec": 3.198214425938204, "model_name": "muse-spark-1.3", "output_tokens": 357, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:17:19.484486+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d7ddded6-af5c-48a1-9ebf-78f64cc35f47", "result_sha256": "cf5fe72f8f594cdac532e7af772486fbf62d7243850df77dff567172c23f88d4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.015867, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 36.02220013702754, "model_name": "muse-spark-1.3", "output_tokens": 2989, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:16:18.911998+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "852068da-46dd-4aa0-8fe5-010760fe5664", "result_sha256": "73e041369c2982614b07003d71b993ffa28a18c2d6dff31d66ac616adc67846d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0058285, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 8.939542611013167, "model_name": "muse-spark-1.3", "output_tokens": 627, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:23.547978+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "5505c829-734d-40db-89fa-e51dea7a988f", "result_sha256": "501b5559c1e1b95f3a92a7ab54e4d95c6ae4e9f9888fb9a281cbde4a0547f352", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009339, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 17.99480169499293, "model_name": "muse-spark-1.3", "output_tokens": 1453, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:17:46.369676+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "20119b59-b3f2-40d9-b92c-f63a2837b597", "result_sha256": "c3f36e67be10331a4d63009c343ba1cd016695c9403b4d2c9de3e821e52025c7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00676775, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 10.128204443957657, "model_name": "muse-spark-1.3", "output_tokens": 848, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:20.850788+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "2e378413-aaa4-4568-bf4c-fd9fcd75be94", "result_sha256": "405aaf55b9796d8ea6922d5048d07a04283bcb6aaba4d5db11dfe90b6dc6800b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00510175, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 4.533221489051357, "model_name": "muse-spark-1.3", "output_tokens": 456, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:11.083784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "b2664c7c-b112-43aa-887d-a5aacbf55be2", "result_sha256": "cbfe302a58044ea6f48b21f91a063f79396dd0d675d9c87fc8278e12560411ae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00468525, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.00944214197807, "model_name": "muse-spark-1.3", "output_tokens": 358, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:23.051006+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d63c9ebf-3981-4b15-bed4-0f59ca058e44", "result_sha256": "988e11cd04abdc90d34ba2f2a8e6fdf5762a4bc6b6fbe7c12a86940c1a5bfafe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0042305, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.71798677300103, "model_name": "muse-spark-1.3", "output_tokens": 251, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:45.734481+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "2e2ee9b6-4abc-4153-ac7f-3f7a544fd048", "result_sha256": "1e6f88d58bb798a116099430f130b53f75837e3fb97581639c7a5c3685ecb0e9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00467675, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.6972280289046466, "model_name": "muse-spark-1.3", "output_tokens": 356, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:14:33.479650+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "370acd3e-f6bf-405f-8df0-fc9a32875da9", "result_sha256": "7c76583b9e6b22ec7fce8602317f141261852f29a315a71e8f1b3f0db2202fde", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004409, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.2296659239800647, "model_name": "muse-spark-1.3", "output_tokens": 293, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:49.833938+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "63e289b7-00b5-416e-acf2-d854362586ee", "result_sha256": "039769262f5fda2240e3443234df16a8bd0da79d498a519afba15a08e7c25499", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00483825, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 4.501608905964531, "model_name": "muse-spark-1.3", "output_tokens": 394, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:26.653401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "89c6f825-bead-433c-a324-1c56be6d5ce3", "result_sha256": "78d2fcc526dce7fb49275a3085996ecc6e7e807204aefe2e2196da5ed7eeea7c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00443875, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.3242592150345445, "model_name": "muse-spark-1.3", "output_tokens": 300, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:25.785733+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f6564566-fa89-4791-ae41-468a20acd004", "result_sha256": "1028166b8f2937e1b149d23c9db38fef74c68e1191205f339149b2ac30c335fd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00505075, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 5.097933196928352, "model_name": "muse-spark-1.3", "output_tokens": 444, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:11:48.825412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "9969a4ca-f34b-47e1-b84e-8e8223488027", "result_sha256": "e4fc5792ce64eaac349a112f41c4569a7970811824be77b8946beea7a2354411", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004401, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 3.037510168971494, "model_name": "muse-spark-1.3", "output_tokens": 287, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:26.426489+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "2585aad3-d751-4fba-8c50-b92a7f509b15", "result_sha256": "9a022fcc61155fa37a68f8325ea75ecf69f4fe01ea36c68bc3f552da446b6b51", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00526375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 4.938053313991986, "model_name": "muse-spark-1.3", "output_tokens": 490, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:09.650749+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf3f064f-0e22-4119-9a18-6182dff66e81", "result_sha256": "aa32801f0188fd3ac7fc2cb0103d09ca17bb379b1bcff9b19050642c6e279cf3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0047665, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 3.8126083010574803, "model_name": "muse-spark-1.3", "output_tokens": 373, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:26.584386+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "5c4dc469-329c-434c-bf94-b08709fb17d8", "result_sha256": "85a6fe26f9642f3346c28a4185aad2a5c25e93b7b5c570287a05e389c2b10580", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054465, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 6.438414352014661, "model_name": "muse-spark-1.3", "output_tokens": 533, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:45.066709+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "c2488b3f-72ba-450c-95ef-6f9f9ceeeddc", "result_sha256": "ac3807a151f62e51288486aa22d3e6f7f0935167eff5d75e16bea868b511a02a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00869775, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 12.015884542022832, "model_name": "muse-spark-1.3", "output_tokens": 1298, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:59.088116+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d2d1b947-597e-47f2-bc8c-087e005846e5", "result_sha256": "5eb933fa5428dd5707107a889ecd34bada53e56ea2beb91ba5fa17b87517b3b3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0094755, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 15.39462828997057, "model_name": "muse-spark-1.3", "output_tokens": 1481, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:16:25.566527+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "bd5c0364-5bdd-4109-ac6b-4744de05a995", "result_sha256": "e44c7bd6208ea6b84b9dca1a3110237c0a32dc5a1c1b9ed7a20038e211f31366", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00602025, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 7.2748046070337296, "model_name": "muse-spark-1.3", "output_tokens": 668, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:51.298113+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "04a06a29-bbd1-4e8e-a456-f57dec8a0a58", "result_sha256": "16f361502e69bd3afec2d553291cfe3b66c2544047ab87092b60a7c9fb09bc2e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01281175, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 22.86072623205837, "model_name": "muse-spark-1.3", "output_tokens": 2266, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:25:01.646420+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "189bafe8-e3cc-4dd2-8b77-a087a1b062e6", "result_sha256": "8d68f1aac607bb9654ead54e1dcca5cdaf47bb2e8ca231a9f823e1e999001c76", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007699, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 11.490154108032584, "model_name": "muse-spark-1.3", "output_tokens": 1063, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:12.004098+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "47cec8ff-8d3f-4c05-be08-b3dd608b0ec6", "result_sha256": "b02a8be287e011c3f7ec208b34788d47386aa1a6c210428c49ece51fdf7b18da", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0055825, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 7.070157867972739, "model_name": "muse-spark-1.3", "output_tokens": 565, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:24:19.537518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4025f6cd-ccae-4169-b968-2ac2345234a1", "result_sha256": "90647fce5ed8b05a4e3ba545d7c9062802be0a5a201b6d5c92a31db985008ebe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00858725, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 12.823870418942533, "model_name": "muse-spark-1.3", "output_tokens": 1272, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:30.272571+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "7507dc5e-6ff6-4049-ac3d-3ebe13e6ba63", "result_sha256": "736875ae0cb9905bcdf1dc89b27e90378185b3c5d5ed8c273fe370c9ee3714f7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0131305, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 25.061665587010793, "model_name": "muse-spark-1.3", "output_tokens": 2341, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:00.924286+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2fc7816-0072-456d-bdc7-c930014f284e", "result_sha256": "c740ea8718ae298d1c3a33a113f71caecede3bb50fc9d93d4624eb95c7a91a50", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00466825, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 4.481364218052477, "model_name": "muse-spark-1.3", "output_tokens": 354, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:58.758362+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "69c5400a-cdbc-4a96-8a70-a3fb8f00bf1c", "result_sha256": "bf69a94236a9f2389f1ad352289ff96bb66cf41eea9173aba72d41324d778cb3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00414975, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.4605207630665973, "model_name": "muse-spark-1.3", "output_tokens": 232, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:03.982897+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "0927771d-de93-4be8-91ee-e2248f9ba085", "result_sha256": "f98b54501bc5c8f3d46be158bbcad0e94ac759066c2c5ac5532287bb478e4483", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004681, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.60226803610567, "model_name": "muse-spark-1.3", "output_tokens": 357, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:25.672990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "6a33229f-da8b-4f04-80e9-74477d6fdcfb", "result_sha256": "1f0db3c76ad1a2aef21f7f70ee97e9a2cf75d3c593cc0d6f8889d526c105e585", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005123, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 4.047441366943531, "model_name": "muse-spark-1.3", "output_tokens": 461, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:58.437347+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "a880f58c-c4f0-424b-9d91-c5e7770599e7", "result_sha256": "e27209ee6d45515b7382d6326204d0ee4c9e1792140d90e1cb1ac484097989ad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00486375, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 5.5223972470266744, "model_name": "muse-spark-1.3", "output_tokens": 400, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:52.266581+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "7bb94433-dacc-42a0-92f4-17f409a111d2", "result_sha256": "1d735b7dfb6a5999cdb3267bb1281d46cfac2eb6cae03f1abb5178b949471d2a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0048595, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 5.154416828067042, "model_name": "muse-spark-1.3", "output_tokens": 399, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:25.493570+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "b62140f4-0d7f-437d-8387-42ffbf6fd0cd", "result_sha256": "e0a632fb4e6bcaf43d6f79c5a60b8efe2b9dfee9002dcce7cfbd5a59ad418b18", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00432825, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.691048313979991, "model_name": "muse-spark-1.3", "output_tokens": 274, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:29.580431+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f69cb93a-85e7-46e8-839f-b17b683184bc", "result_sha256": "5ef9a39f8479f87bb03f0b5b9d2c5b032e9bb9569e7d08fc738c0384686756c0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0043835, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.9603804970392957, "model_name": "muse-spark-1.3", "output_tokens": 287, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:48.678555+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "54584cf8-890a-454c-a34a-c4eb12b26675", "result_sha256": "df8f050cfaa935d31f16bb2b4e02e23bf49e1b9a37d009f63cf9a11e232a0b4d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00437925, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.4247809859225526, "model_name": "muse-spark-1.3", "output_tokens": 286, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:12.003098+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "045e17c8-7992-4a6a-908a-31c5acdfae9a", "result_sha256": "665689fa6428f04338677aefc700fde74224523447207664ba35bf9d7c984ac0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004698, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.630960409063846, "model_name": "muse-spark-1.3", "output_tokens": 361, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:16.622465+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "26dfb02c-781f-4705-b7fb-a3159478ab26", "result_sha256": "4e44aeb9f6bf7112b2c98b97968e2bc1c5f9ff64807e5520203cdf55d8d1a9bb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.003882, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 1.9509597410215065, "model_name": "muse-spark-1.3", "output_tokens": 169, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:35.568257+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "0cad11f1-c03d-424d-b17e-486b554ce555", "result_sha256": "c35b68e9bd92bedd0e08165a7f8697a362d670b4fe68a8f5c3ab79f3867dd872", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00482975, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.74847200489603, "model_name": "muse-spark-1.3", "output_tokens": 392, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:15.983095+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b5afd44-aaf2-4393-8e34-dd4c36c24b16", "result_sha256": "4eb126f3bf38d3a34a5b6d72c38d982f4151ddda3716e7b6311447fd05b0e852", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004851, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.680411819019355, "model_name": "muse-spark-1.3", "output_tokens": 397, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:49.423490+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "9dcd413d-3d08-4184-b7eb-49dd7d1aade2", "result_sha256": "44bd2ed93577db99e80698815c1527a8a058afe716451e7eb065777b4206453a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00500825, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.5806955749867484, "model_name": "muse-spark-1.3", "output_tokens": 434, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:37.587692+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "19281de8-103c-433b-9be0-3f2664b88a0a", "result_sha256": "142b87984ce91a5cc3b7693a721f966a8b754157c600eac6006470e68d6ce0b9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00437925, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.6597149239387363, "model_name": "muse-spark-1.3", "output_tokens": 286, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:42.352152+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d5523abb-870b-401a-88e2-47746424e715", "result_sha256": "3af5e660234ecdeefc4df5f1d0caec6b2dbe5ab83bacd38d3c9e04e55eccc8df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0042985, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.179440753068775, "model_name": "muse-spark-1.3", "output_tokens": 267, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:35.275439+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "98279b9f-e0ae-4d0c-a81b-d73920fe99e8", "result_sha256": "d9d6e39dc3554afa51a169fc9d04430acaf12e0934885adddcfc0ecda1420969", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004137, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.6075911769876257, "model_name": "muse-spark-1.3", "output_tokens": 229, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:19:14.994636+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "be702ea4-617c-4f64-8003-38df64b7f5e5", "result_sha256": "959399102fe200a8d056c0aee092723cad9c045d0dcf6dfcf1d5d93f3978edd4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004698, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 12.856390536995605, "model_name": "muse-spark-1.3", "output_tokens": 361, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:28.131926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "bac10ee5-348c-4608-ac5d-2d78f2a7a38d", "result_sha256": "4c39b25d710814e78400dcc77c8c242ffa266495d9bc4abd9397d35511e0fb70", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00431975, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 3.332331531913951, "model_name": "muse-spark-1.3", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:57.258446+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "9eabc0f0-a1d0-49eb-a9fb-7b11f7c7583d", "result_sha256": "40c61777896f9ba2d7b33f802e47b053caba626c7ce40f46a34731e657b41c74", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00415825, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.3348857139935717, "model_name": "muse-spark-1.3", "output_tokens": 234, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:48.611398+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "a21c92cf-82a5-45c7-a61e-eb1751449f1a", "result_sha256": "ee52d599843aaddbb1772827766f6a9e306b86c9f2ee8a21159fadc4d7d21f7c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004239, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.375518771004863, "model_name": "muse-spark-1.3", "output_tokens": 253, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:20:31.416713+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed31b28c-360d-4d7c-b7f6-edb49a02df77", "result_sha256": "7d91b2c5b5269b87d59a1dffe376f87b55899d10f9220521ab9e3afdffa99ce1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051145, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 5.008050454896875, "model_name": "muse-spark-1.3", "output_tokens": 459, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:17:10.389778+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c7cc464-5e26-4f01-bc74-b14d3e76ff77", "result_sha256": "497a5af30103adf3de84553a4e726bab1b3b3b87d4cec1ff6503ecba35227d71", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00419225, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.6534700450720266, "model_name": "muse-spark-1.3", "output_tokens": 242, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:40.443236+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "6484e6b7-5cbc-4079-a60f-c70edf322bf1", "result_sha256": "68a70fa397e023c6204b29243eee2e89c26e101814fdca5ab6116c899fa813b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004443, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2531, "key_errors": null, "latency_sec": 2.8763882409548387, "model_name": "muse-spark-1.3", "output_tokens": 301, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:15:08.974274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "732f88e5-3495-4dd6-98f7-dc434369e735", "result_sha256": "e87c5ceb457be6c941978dd106f9700a2d05fdcaf1c131cb4555a8c54d0b8ea6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0056625, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 7.7919144129846245, "model_name": "muse-spark-1.3", "output_tokens": 585, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:29.977826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "211eb077-009f-4c92-bc39-33e1ca3389ba", "result_sha256": "d072185011f60811a3651a4e33ce3476e194275398ee3b01a7797945cfb16168", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00578575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.272609345964156, "model_name": "muse-spark-1.3", "output_tokens": 614, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:35.634536+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "df37f85f-5ec0-46db-acac-21cfc9fcb549", "result_sha256": "fa4d9c76fb29cb1b3c8e64fc334738f57fd39be3c118524fa1bb592268dbc554", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0051185, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 4.803880111081526, "model_name": "muse-spark-1.3", "output_tokens": 457, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:33.843642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "92db5a95-c1d1-4d2d-9735-6fac7eb0f007", "result_sha256": "ce7bcf4e2287fe6806e24eeca261cb6a776f74cd4ab2de9dcb20da1e5152ff9b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00593025, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 8.583682957920246, "model_name": "muse-spark-1.3", "output_tokens": 648, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:29.157958+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "5094e50c-1ce1-4b0b-90d8-a7377e5e0bf9", "result_sha256": "4901607900507711c07dc166ac829d25a39f22f124888bf62b3c6a7a7ac45c36", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01076675, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 22.374490175046958, "model_name": "muse-spark-1.3", "output_tokens": 1786, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:09:25.929573+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e58caf9-0b2a-411b-8d0f-04495ae9cd2e", "result_sha256": "11d8850ce3b307757143d0542ed58ec6d4c27263cacec0b9c4d51297385d7462", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0065295, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 9.013796739047393, "model_name": "muse-spark-1.3", "output_tokens": 789, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:12.791926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9a0d709-2886-48e0-a8be-06c5b1f49806", "result_sha256": "7d0aa6553fbeae79e5c26845e78ec6c89fc049a73598274851bae6e8e4e498b3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00709475, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 11.219408186036162, "model_name": "muse-spark-1.3", "output_tokens": 922, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:45.171362+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "13d02ded-821d-4b2c-aff7-83d4270a617c", "result_sha256": "f698c09840c3cae17a4af13b495d6e9f250ed297da07668e74a16dce1dc13e55", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00698425, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 13.6606033280259, "model_name": "muse-spark-1.3", "output_tokens": 896, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:12:37.627833+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "8bbd8b04-88cb-4943-8845-56730934f37a", "result_sha256": "52498e2242a53c57d169d4d09ff30642de232d16f40f6570366aaf7b2f468573", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066995, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 8.476745955063961, "model_name": "muse-spark-1.3", "output_tokens": 829, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:11.323363+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "0ceab312-72d0-49b6-ae02-2462fd2b93c9", "result_sha256": "b6ebe9b1f8fb69b38cbaf8bc5297d211a15bdefcd2a80cf4b1c1ecfe1fca4ffa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005875, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.52566443500109, "model_name": "muse-spark-1.3", "output_tokens": 635, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:10:35.542285+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f0c758b4-09a6-48cc-a739-3c9b70673889", "result_sha256": "546ff1f3ba9faf994d5151d08275f563b3c5b931d76a8e494df1573458ee7955", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00553925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.068970128078945, "model_name": "muse-spark-1.3", "output_tokens": 556, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:23.651851+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "e24370fb-9bfa-426f-a3dd-81ff126d0f23", "result_sha256": "6b17bf2b37cb9312dfe0317f2f65fbcc9a99a08aa253da74dc92287c672b04e6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005943, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 7.769846466020681, "model_name": "muse-spark-1.3", "output_tokens": 651, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:21:52.466350+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "95d7fe63-275a-4d02-b20f-13e6a99db9e7", "result_sha256": "3503034cfa14751d61f902fc96d717af7750a090bfb7fec832813c5b41e6434b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00579, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.410999399027787, "model_name": "muse-spark-1.3", "output_tokens": 615, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:24.614090+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "16e59de9-7021-4680-998a-f899f59e5bc4", "result_sha256": "42f60a753622b0d0077b8eb1e93a658fd1c852104eb6ec0a44850a474cafd56f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0062405, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 8.159184337011538, "model_name": "muse-spark-1.3", "output_tokens": 721, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:26:27.413445+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "75474af1-eb69-4da2-905e-752e072d636b", "result_sha256": "74515cd4aaac7017266a3a390c6bb95ca3976a536b9b7764bcb34b5484623748", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076005, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 10.676851279102266, "model_name": "muse-spark-1.3", "output_tokens": 1041, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:46.761648+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "816e3625-6c9d-4ba6-8e89-f3eb676b9db2", "result_sha256": "75df60ea674b556fe6e881f1db2301836efd26f9e93b195363f3490a34774455", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0078725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 13.822517538908869, "model_name": "muse-spark-1.3", "output_tokens": 1105, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:21.345887+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "1ca52d72-df41-40f2-99fe-ce36647a367a", "result_sha256": "1b720df472ab2cdbe5d8b02465dc2e773e2afb9023ce692139777b5daef38b3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005399, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 6.209897020016797, "model_name": "muse-spark-1.3", "output_tokens": 523, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:18:28.930938+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "32fc2498-436f-4ca1-a041-dcb8c786bab7", "result_sha256": "c09832ad3d28f18c92e1708622815e517332f888fbc0912151a845b529e9c844", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0054925, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.296266062068753, "model_name": "muse-spark-1.3", "output_tokens": 545, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:36.321052+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "01bbe97f-5f7d-49a9-8a6a-ff99df712614", "result_sha256": "8b45b6b88b697db78fc95a6c5654ff70be5a748b52b3741c3526db42d8ca435b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0097935, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 18.074276293977164, "model_name": "muse-spark-1.3", "output_tokens": 1557, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:25:42.367130+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "345c59ab-6700-43fd-884b-aa90acc62017", "result_sha256": "909ad5d28b47891b62d3dc2f2cbd1696cfdf0f7cc0691db30d9b3b260080c763", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00668675, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 7.742811545962468, "model_name": "muse-spark-1.3", "output_tokens": 826, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:19.458162+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "504d3770-3572-4124-8436-cd64d6d3695d", "result_sha256": "96704b1b9901a2b7866e94fc69ba3651010e931587d10e01697a34337ce85e33", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00586225, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.864560827962123, "model_name": "muse-spark-1.3", "output_tokens": 632, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:52.824508+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "20ea34eb-42f1-4839-880d-4423a34bb71f", "result_sha256": "55e2b3841c14b895f5c7327eba5ed7115920618fed6b5eea1aab4b618205b89d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0082635, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 14.070832891971804, "model_name": "muse-spark-1.3", "output_tokens": 1197, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:12.576009+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "9ce33f96-b0ed-4385-a211-8b2ce3529f45", "result_sha256": "e6d197cf60881fcc2a91ca220897e25808c4dfd91f822e3163c50dc14462d57b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005501, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 5.847028578980826, "model_name": "muse-spark-1.3", "output_tokens": 547, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:33.696518+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "92e52fae-5e84-448d-87b2-8453b7c93ae5", "result_sha256": "c918f9a57c85941b86be6ba8e41cca1ed1c6d28ce96e6b7ab1064ea3ae0d5ed8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00669525, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2541, "key_errors": null, "latency_sec": 8.698116454994306, "model_name": "muse-spark-1.3", "output_tokens": 828, "prediction": { "choice": "C" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:16:57.736835+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "b9e49825-d6ad-4ca5-9cef-5936ef13db2e", "result_sha256": "0e71382cdca020e4430397c42f4c6162237cca4c369e3b33faf72fafb3a2aa83", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0080485, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 14.1977550330339, "model_name": "muse-spark-1.3", "output_tokens": 1142, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:21:12.514035+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "62bae608-f073-424e-9e87-06d9da46eaf8", "result_sha256": "17b0dbc5b94a0266a172a3508fab4f97187e45cee077f5824144b64b1b408134", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006816, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 11.227890470996499, "model_name": "muse-spark-1.3", "output_tokens": 852, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:14:30.219711+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "8bd1cba4-3ba7-4ea1-a85c-b617bbccd18c", "result_sha256": "45c6f5e14df8e5774cd4e7f3b89cdf33bb666b915bce9cec30d66b5acfe23624", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00603825, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 8.687176381004974, "model_name": "muse-spark-1.3", "output_tokens": 669, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:20.452183+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c3123a3-2617-461a-a5a3-e8e32521626f", "result_sha256": "e32f8167e4d563a63a1f117616c697059d54f8e3b9d3fca4f62867be13824833", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00631025, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 7.220046381931752, "model_name": "muse-spark-1.3", "output_tokens": 733, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:08:54.278009+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e014794-c660-4ae7-875a-9b62c3db59f5", "result_sha256": "1dffa6804a8ef9f919e4b6c78d66da7b710a49dfc9bd996a059dd6c1e7708ca7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00665875, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 11.052655894076452, "model_name": "muse-spark-1.3", "output_tokens": 815, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:38.299228+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "002c4e05-3bd5-43f3-8d8b-1b2399e1066f", "result_sha256": "534977653bc7d2d1a5c72f1ae562be44458b768b2f60a70405599e5c40840c05", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00507775, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 10.016927120042965, "model_name": "muse-spark-1.3", "output_tokens": 443, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:27:43.145125+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f802c179-c14e-4039-81a9-c1e341dbec04", "result_sha256": "db633fe36ce15c6cbba0f674d1e584eb2debe2116a331db4eaddcfafb0c117ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00580875, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 6.863545258063823, "model_name": "muse-spark-1.3", "output_tokens": 615, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:20.704359+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "8fef2b49-f3a5-4295-9274-a8a9978758ae", "result_sha256": "e6d467dfca3cb5c26480ea706fe7fe707677cf31e4fbd86ed0226e8ba47fec02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005082, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 4.326736490940675, "model_name": "muse-spark-1.3", "output_tokens": 444, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:16:48.675372+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "7c9ab018-1ec3-4a3e-bb8b-a809850a265d", "result_sha256": "b2ccdb43d7d78893dfb8f387d890fc25261df2ef4505a08b62bac7a16b2058db", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01392625, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 35.349566776072606, "model_name": "muse-spark-1.3", "output_tokens": 2525, "prediction": { "choice": "B" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:18:14.546816+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "48e3e12b-836c-4bdc-8245-79786d23bac0", "result_sha256": "b47f4a9ef05234a25821aed80c651bcdb6f17b636bcdef255b0f964be45d7758", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.006714, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 9.743025718024, "model_name": "muse-spark-1.3", "output_tokens": 828, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:19:57.360745+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "dee6bb8c-f83b-4645-80e8-66e2b0804ea1", "result_sha256": "2956573aa84e22ce2be71946849abf65f6dd77727e8309f6f80e29efbaad212c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02023325, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 47.48063681402709, "model_name": "muse-spark-1.3", "output_tokens": 4009, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:20.803439+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "130d6cfb-b23d-4dee-b7ab-792085634c07", "result_sha256": "8dbf6d819ef305f61c89675230be7f7b0095ccd3d7813ac01994a25f9330b532", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02066675, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2556, "key_errors": null, "latency_sec": 52.52634105598554, "model_name": "muse-spark-1.3", "output_tokens": 4111, "prediction": { "choice": "A" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:12:53.777730+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "98a6858e-3c3b-4be5-8f90-928a3a82184a", "result_sha256": "58da9311f2f6f01e2711ba98d76e370cba892bbbb9ea14570f0144368630a83a", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01285475, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 23.464460386079736, "model_name": "muse-spark-1.3", "output_tokens": 2272, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:26:14.496155+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "626dca65-9e34-43c8-87db-d5d7fec58a22", "result_sha256": "69f0638fe8a1e82979f915b7d95cc0e20c67a0554e60cdc29571f755a6a06950", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.02241725, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 57.25740506500006, "model_name": "muse-spark-1.3", "output_tokens": 4522, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:25:45.214357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "fbad34bf-7e4a-425e-be10-63a962c6a479", "result_sha256": "6eaab2ee28d06467e06e1c13411a31186d656c77f62c3bba22dba4b3d35089ce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010989, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 18.39863368391525, "model_name": "muse-spark-1.3", "output_tokens": 1833, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:06.082628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4c8f650c-06b6-40e7-a0b1-537f34ad433c", "result_sha256": "ff61c93054bb9c9fc8a9ba2a8d57d77fc02721a75a924d159511dba42d2dcf52", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.02359025, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 48.129503038013354, "model_name": "muse-spark-1.3", "output_tokens": 4798, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:28:03.036188+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "1b2e6d9d-8d25-497e-96dd-d15a3ddde346", "result_sha256": "11dd24ae4e33b97fa9951c684b467e6ecf288d04b9908495182a441476a90f1c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01180925, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 21.042858717031777, "model_name": "muse-spark-1.3", "output_tokens": 2026, "prediction": { "choice": "C" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:11.850117+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "da435098-3a9e-4b54-9e17-466932a260b1", "result_sha256": "56c02db482d0c836ae8429ea30e6bcc11d023ed349e678299ac4ca19489cbbfa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01121425, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 16.180186784942634, "model_name": "muse-spark-1.3", "output_tokens": 1886, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:14:54.375193+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "393f34d9-cfee-4cd2-87fe-570bbd700bb6", "result_sha256": "3b169ba18eb6ca2852e500da0103692f58c68b0656d2749582da7d85c394c435", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012757, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 18.30742082395591, "model_name": "muse-spark-1.3", "output_tokens": 2249, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:32.942267+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "a37e0c75-5b10-4c00-bc97-6ded71519e7c", "result_sha256": "19ea6f9d34314de6678fa067708f663e0bc8f646db5cd3503f39e42c8e3c1c22", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0142105, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 23.391514430986717, "model_name": "muse-spark-1.3", "output_tokens": 2591, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:03.340742+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "1f96798e-cac8-438b-a414-5aaaa0188872", "result_sha256": "19a818e7f9dcaf792c5698baa9b2f860ab70e4ff2871663166636e674dc8d29d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01316075, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 26.64600040204823, "model_name": "muse-spark-1.3", "output_tokens": 2344, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:49.319899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "26571765-53b5-4754-aa7d-29b2a709ee7b", "result_sha256": "9e9958b5632dca61124f04e7ec46ba26ea1e9b60a6ee9dbafefe14849d5d73bd", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.01055975, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 16.30803009599913, "model_name": "muse-spark-1.3", "output_tokens": 1732, "prediction": { "choice": "A" }, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:09.888271+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2f85fc0-a84f-4229-ba80-d1fda09f1146", "result_sha256": "d3630a6178033cc0ad66b9061e53bc6520cda160e41fd0552064812eee1644fd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0230165, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 44.137095592101105, "model_name": "muse-spark-1.3", "output_tokens": 4663, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:13:47.064068+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d280c4d2-a69f-4b64-9b0d-e49cf033ea0a", "result_sha256": "a6bd52099608e8fbd8985969bbda3121833c60d71155378ac2f2830d755bb17e", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0135305, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 21.903505982016213, "model_name": "muse-spark-1.3", "output_tokens": 2431, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "openai", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-17T03:27:51.330736+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "71d80a9d-b8ec-4042-949f-8832b584f7d3", "result_sha256": "5ca484facfd5881309f5b364f527824385c010b13a8a02b03e5b6151f318d7be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.013981, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 24.60176968004089, "model_name": "muse-spark-1.3", "output_tokens": 2537, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:23:30.227801+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "66c39474-f3fa-4023-b860-6988a34058d9", "result_sha256": "13fcee986f4563f19290b50e9096e6ce7c186d60a4af9ba53a1effd3f6a978cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0090085, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 13.689074197085574, "model_name": "muse-spark-1.3", "output_tokens": 1367, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:14:21.438165+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "85d769a9-1792-4ebd-bd30-6004fef4431c", "result_sha256": "1f69afe56abafe38185a2436c228c4330e4cb700a1d27f231ad375b0cb3f8524", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.012298, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 25.742531750001945, "model_name": "muse-spark-1.3", "output_tokens": 2141, "prediction": { "choice": "C" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:22:03.648065+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4879b7bd-bcea-4d81-9e32-b665e3bea181", "result_sha256": "1a5b1bca1ec8ac8e02a2775958050eabe94cd59a343de06f532999a5e3ccc8f6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0065945, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2559, "key_errors": null, "latency_sec": 7.244312442024238, "model_name": "muse-spark-1.3", "output_tokens": 799, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:13:05.774396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "a99522e1-a292-4340-9bc4-56e39e683845", "result_sha256": "79b4dd2f56fbafd2edc4f70fee19f6d0d6af3552fa2abfdd56133354a1b18506", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004702, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.6712237850297242, "model_name": "muse-spark-1.3", "output_tokens": 369, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:56.894814+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4401531-a7d8-4747-ab0f-a14d4d14ebd0", "result_sha256": "2287a62ce6fc03b1ddb9e3693fae6433e5a75bd58a1b48edfe8fad82f5cb8f0e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004923, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 4.02433046803344, "model_name": "muse-spark-1.3", "output_tokens": 421, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:34.550898+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "0fe386bc-065b-4749-92ef-abe4dc3d7a49", "result_sha256": "fb9316ffa189e49d884fe2d35e20b627ff9a8ee666912e1a2dd09bcfd667be8a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00482525, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.746577135985717, "model_name": "muse-spark-1.3", "output_tokens": 398, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:20.510708+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "620e8b11-17be-4c2e-953b-387462a13019", "result_sha256": "9b793c51b8b9660cff5a191666a13bf7f66cac3bfd2fc54e8b32c923a605e327", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00563275, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 5.527881628018804, "model_name": "muse-spark-1.3", "output_tokens": 588, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:32.354769+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "13b313f2-b5d5-4cbc-b538-21eb7875b9c4", "result_sha256": "0d52a6bd39e659883fb676f9d68bc2b70982bd6e6b0c89997579567f7de8538c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004396, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.8758477160008624, "model_name": "muse-spark-1.3", "output_tokens": 297, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:24:29.134923+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "3059c0c3-e063-48a9-8a2b-510097136320", "result_sha256": "b90319e38dba2a608ecb2295f4487b44ceb9b93805e6946a8e0f097060559609", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00502925, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.0977998620364815, "model_name": "muse-spark-1.3", "output_tokens": 446, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:17:36.475286+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "273cab93-8860-4b06-a738-1400629658f5", "result_sha256": "8ce8e49af9643bc8ff3d234c5320b745b3f7d21dbeafe9d15241d822e982972c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00465525, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.1411209820071235, "model_name": "muse-spark-1.3", "output_tokens": 358, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:26:10.939889+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "17d3ddc9-c758-4d25-bafa-a866ad414139", "result_sha256": "e074bfc2efdc43c51c334f1a7e6f9b9302e897a2fb83da15ec3cd47627ff257b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0048125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.974001520080492, "model_name": "muse-spark-1.3", "output_tokens": 395, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:09:50.567926+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "405a95fe-c57f-4563-938f-8a8514b59b86", "result_sha256": "0d7c51c7062607c9a4f9d5497f9cdb143a4cdbc0a800c4d7e2a8789cfc74fd3c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00417925, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 2.9343231569509953, "model_name": "muse-spark-1.3", "output_tokens": 246, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:02.877684+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "ae5337ca-fe4e-4735-b79c-36ab2511da7e", "result_sha256": "cb4708400b9d452c3d1c5fa4831f0bebcaea8155a3f1a94405621d03041541d0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00431525, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.469685825984925, "model_name": "muse-spark-1.3", "output_tokens": 278, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:12.708650+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "171bc9f0-4a29-48e9-b33f-ba0203c42869", "result_sha256": "b274d9a8985b35f2ae2d7bca2a99b15ce05c3cfe38cf5fc5ca3e450b3634baae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00428975, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 3.601227528997697, "model_name": "muse-spark-1.3", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:50.763050+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "55131788-69a9-4ded-9a2f-d569c2f9cf49", "result_sha256": "d3d29cedab3b278a0219e3ef13d20d4c8acf39dbb2dd51a30a24c87ddacc66e3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0050165, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2507, "key_errors": null, "latency_sec": 4.0966760700102895, "model_name": "muse-spark-1.3", "output_tokens": 443, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:23:01.883892+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8baba5e-28e7-4be8-9817-50b4727ab9a8", "result_sha256": "7b24bd7ec9c187a9682ddb1b9354adc8fb7b2599fb6ec476da1eb427978504bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0040185, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.8032652259571478, "model_name": "muse-spark-1.3", "output_tokens": 202, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:21:50.534930+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "351b0ae1-ccd8-49f7-a76d-9ff53538627c", "result_sha256": "34069d6ce4c890bf6adc140aea42025e23adb32040ad900a4fdee2e74ea45260", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00407375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.413697247975506, "model_name": "muse-spark-1.3", "output_tokens": 215, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:23:16.454062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "10fe8efd-20fa-4376-8aca-521ba04a0ac5", "result_sha256": "9f921ba09c0521b852a0b99ae8618fe2e7e89cc9c89c252b430e7eef22d88f76", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00435425, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.771300456020981, "model_name": "muse-spark-1.3", "output_tokens": 281, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:08.242913+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "bad0c73b-aaac-4987-8560-d88eca842c06", "result_sha256": "5c80cd80ab575669af287e5a267e89cf7b84715937b87b31386d5946d30ff74a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00368275, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.772790111019276, "model_name": "muse-spark-1.3", "output_tokens": 123, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:09.898239+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f0bf37c8-aca8-4f71-ab5d-598226240a6f", "result_sha256": "2cc25a52d037a170f736df56a2a0b850478d2171c324a227ffee19d1a79e250b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0039675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 9.459931844030507, "model_name": "muse-spark-1.3", "output_tokens": 190, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:30.209062+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "f4a36a24-2a06-49cb-828e-934c1a846682", "result_sha256": "d667c3875670805676afc951d0de9450468f84e0d8fec42453ebdedf1d307752", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00407375, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.389686929062009, "model_name": "muse-spark-1.3", "output_tokens": 215, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:14.688357+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "7ec13e07-cbbd-43c9-bd59-51ce78100b08", "result_sha256": "99c52ff0b318f48402d6815807628b5c457e3df5097e2ef8cde1db56b9e837be", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00393775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.9811276539694518, "model_name": "muse-spark-1.3", "output_tokens": 183, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:24:43.628134+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "66c31067-eb95-438e-b39a-2de975193bce", "result_sha256": "5ec1a6aaa9013574d78bf3c3ab49518958121db9a22ffbce9f06355c378bd70c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044265, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.485293519916013, "model_name": "muse-spark-1.3", "output_tokens": 298, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:22:54.248628+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "691aa402-002d-4450-81ca-48fefcd65492", "result_sha256": "9714087cfc5da4a6a7a156bc5e43a96bdee8f1df4c479674fd4453ec0f95f06c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00457525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.965552462032065, "model_name": "muse-spark-1.3", "output_tokens": 333, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:29.133859+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "dcc2f498-0175-42da-a61f-4fa4866d134c", "result_sha256": "fc996d05ba32d3e1dbe932412edb07c41c153a83ab60cfce527f96e70e9ee87d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00404825, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.576402386999689, "model_name": "muse-spark-1.3", "output_tokens": 209, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:29.687154+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "cf842224-5a7e-4423-b06a-1c1adb2588cc", "result_sha256": "e87841cf3b7a4368cddef4215f03c0bb805c5da2dd89b5278054621e60c30860", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00364025, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.322725179954432, "model_name": "muse-spark-1.3", "output_tokens": 113, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:52.578038+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "025f441e-a591-48b8-9ac2-d8d48718b56b", "result_sha256": "9e4611991d8994deef3efb0de79d7a6d6fac3443d0588bbfeec71856e4c7fed1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00460925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.792696001008153, "model_name": "muse-spark-1.3", "output_tokens": 341, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:39.923521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a44cf74-9eee-41ac-9414-df2240b5372a", "result_sha256": "38039978f98f5678fadde8e22664ee50e8d32298c2c4d9507833ae1d7650edd4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004639, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.835718263988383, "model_name": "muse-spark-1.3", "output_tokens": 348, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:13.721309+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d5479fb2-e5fe-47b4-aabf-50707d3443b3", "result_sha256": "22cc0a402dbb530ef55a116be4fae0ae6e001c304368827fd293d0532ed1fb66", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00471975, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.389003223972395, "model_name": "muse-spark-1.3", "output_tokens": 367, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:18:39.586063+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "c75f7210-b8d5-4e0b-9ca9-bb24c549136e", "result_sha256": "f345d62d4039c82e38dfeb2ccd4b85d67fdb5fd2620135c3ccaf910bf71e9269", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041545, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.4012752659618855, "model_name": "muse-spark-1.3", "output_tokens": 234, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:10.021119+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "91933fd7-6e44-4082-a495-720bcabb34ba", "result_sha256": "18aca21f5b53657bd5ba2785bbfbf36076b44417872cd5b42d892c3887c5457a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00408225, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.6552431270247325, "model_name": "muse-spark-1.3", "output_tokens": 217, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:20:19.714884+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "a183abb2-6ae2-4f3f-b6fb-c1372ed25eb1", "result_sha256": "3d5ea2d84c85ac9202b6ce8ee4d470a86c6d53c4c0665de746797bbfbdec1c62", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004163, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.057666338980198, "model_name": "muse-spark-1.3", "output_tokens": 236, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:27:19.983022+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "7416d22e-8096-485b-b736-0d505aad24c1", "result_sha256": "4ac60abae353e9896f0adcf0dbf70ab301ed6f39d479f5653ef5eea7b2326dc1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00393775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 1.5903011920163408, "model_name": "muse-spark-1.3", "output_tokens": 183, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:19:44.786394+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "12a9ed0a-347d-4bc9-934c-4dc0fefa29c7", "result_sha256": "51ed6595e83fc9bcbf9885f111582fa53669ea1753dd9455d768ea2f630a4ce2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00442225, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.979972489993088, "model_name": "muse-spark-1.3", "output_tokens": 297, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:24:00.739584+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "21b349da-80e6-487e-96f4-b58b16589f7b", "result_sha256": "fe581108c8d6653fb12f4a2a2b3a1ee5218044bf16b64d45d47c830f64b69180", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004197, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.3229895969852805, "model_name": "muse-spark-1.3", "output_tokens": 244, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:17.717115+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "8ebcde8f-43f5-4c36-97e3-b8cfbdb54987", "result_sha256": "730045644f37d723d0df97e43c3277b38fa48960a812206f6194ca21e809b4bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0042395, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.840854838024825, "model_name": "muse-spark-1.3", "output_tokens": 254, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:23.580915+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "99a748b6-57e8-46fa-9623-5eb7eb968e36", "result_sha256": "fcdd6f6b6f800d4e3c800d079f865d445e8e4554a0cf1e3d969b2a535f77b086", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004265, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 2.1232223669067025, "model_name": "muse-spark-1.3", "output_tokens": 260, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:38.549781+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "612f826e-9255-4295-b136-a24d85aaca5c", "result_sha256": "3041877d7ccf92afcdad811bb694055af7ebea8c98c1196a5e9317d1319a852f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0045115, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.312083658995107, "model_name": "muse-spark-1.3", "output_tokens": 318, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:22:28.997929+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "7743e248-177b-420a-a502-858e1c4b69c6", "result_sha256": "ac3cc3317ec8a6e6d05bdebeed9665eb21f615ea8ac1bb9c1c2d362d1f37bbc3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00454975, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2528, "key_errors": null, "latency_sec": 3.5950313940411434, "model_name": "muse-spark-1.3", "output_tokens": 327, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:11:27.110919+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "aaa88c97-a41c-4602-9c02-b401d917a354", "result_sha256": "c679b325dc38da93249b1f4e0e55c4ba88da18dcdf40e25e7049e555ee5df526", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004215, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 2.397384680924006, "model_name": "muse-spark-1.3", "output_tokens": 250, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:23:38.442212+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4fd9958c-879d-480e-868a-96ed56fa5588", "result_sha256": "ffb2c56a53e091e9d9e08594307b1e149b5c791ae33f4c02f97b33b2ad762dc3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004436, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 2.9013583039632067, "model_name": "muse-spark-1.3", "output_tokens": 302, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:22:44.845715+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "407c5441-d160-4ff6-ac99-e9a7d3a236d6", "result_sha256": "3b2e70e0e1ad777df10b5a07cd5d217f2b59f806cc4b767ffe7d60bbbe639db7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0041725, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 2.606167222955264, "model_name": "muse-spark-1.3", "output_tokens": 240, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:03.804197+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "14241b64-7707-4c84-aef9-bccc2b8a13d1", "result_sha256": "263003f6fec4d88e95644aaab6e4ee9d457a0638f2de3f6287c578182f99d0d6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004521, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 3.071595216053538, "model_name": "muse-spark-1.3", "output_tokens": 322, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:24:06.860123+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc79bd8f-6bf4-4a4e-88ca-d73fbf80453c", "result_sha256": "7a36cc659e970634c0f3ead045ee6006de57b6df2684c51dc3d47c3f0ee2eb13", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00466125, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 4.3244752519531175, "model_name": "muse-spark-1.3", "output_tokens": 355, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:21:18.705875+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "bc687b50-b19c-4e92-a16f-63d1768e8116", "result_sha256": "d7896dec95767919fcc7909b468701b57a1506df76539e4696a1f9ca13541a52", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0037985, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 1.5868755269329995, "model_name": "muse-spark-1.3", "output_tokens": 152, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:27:22.380221+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "cac87110-3e02-407b-a8d0-fd46d079bdd9", "result_sha256": "7cc75bb5fe5f087a69d899127f1f1d31d27f104224ddb3ab1c26c7b88d08315c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00477175, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 3.8446933729574084, "model_name": "muse-spark-1.3", "output_tokens": 381, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:10:06.095276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "47afd9ef-1eb3-47a8-b615-2d0253b0f314", "result_sha256": "cade49af775abfe9c4c0772b0b8352c46c64cad531597b7baa5f49eb929306e5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0044615, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 3.4980593159561977, "model_name": "muse-spark-1.3", "output_tokens": 308, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:11:21.395363+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "fcfbb634-37d5-4bb4-9754-add9201a48ea", "result_sha256": "ec3a028eb3455952a7c01e846c2e3c2a267ec314749de9c9c3513954355e80d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.005133, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 5.425197658012621, "model_name": "muse-spark-1.3", "output_tokens": 466, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:20:04.460793+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "c8f9c8a7-4fce-4a5f-8050-f2a3295aafbd", "result_sha256": "ada9a4c0041a06e334df7cbbdb44637d0cd60e3cd52f37472bc77841fcecbb90", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00465275, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 4.06835527101066, "model_name": "muse-spark-1.3", "output_tokens": 353, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:25:38.977691+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "516cfa03-b361-4d38-a7df-4c258d8e4653", "result_sha256": "c303008146d4d20d26e463a6f17f0f2965405d0053492bc180fb21f46857ed62", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00538375, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 7.852506910916418, "model_name": "muse-spark-1.3", "output_tokens": 525, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:20:51.200040+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "2dda7a39-f4a8-4d60-a11d-015cce0da727", "result_sha256": "7e1521f41114208f1d9e6e6b2e9f066125726bcaa9f9da3c985d3f8f6357a0f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00540925, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2522, "key_errors": null, "latency_sec": 6.069431961979717, "model_name": "muse-spark-1.3", "output_tokens": 531, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:20.838916+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "9c01b575-9e72-4494-89a4-7e6d85e45c4d", "result_sha256": "9e04be6ce32dab3adb65e2472668b2a408e181502ff2f70258986aa02a27c36e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0056445, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 5.909804499009624, "model_name": "muse-spark-1.3", "output_tokens": 584, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:09:03.519059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d037c66b-a79d-4e9a-81ff-9ed49621aa6b", "result_sha256": "c07f76696b1e31c30389fbb7d7fb4b510cca55dbe4bbf5a7a56fd39d95685e76", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004939, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.810643221018836, "model_name": "muse-spark-1.3", "output_tokens": 418, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:20:36.692422+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "351dcf77-05e1-4d27-8d50-9c3b0c11b7df", "result_sha256": "84e7c5e8faefa948f0061a250ed97e3d1765f1a0d8d0cbeb47724c7a8d1c5a27", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0050665, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 4.456694120075554, "model_name": "muse-spark-1.3", "output_tokens": 448, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:19:03.232286+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "c85867a1-4933-4a91-9519-a27d960da0c0", "result_sha256": "71ffb4570027a2d1ed59b2ee367c1424aff9db5464747ff3150baae1a186ed56", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00582725, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 5.971923682023771, "model_name": "muse-spark-1.3", "output_tokens": 627, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:15:40.726646+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e659715-14c8-44c7-b1ea-bbc95a757c23", "result_sha256": "8065291a2e73eff49138b202fa4c2a7ca5be15a25e95d955847788eed31092b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00532575, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 6.100056966068223, "model_name": "muse-spark-1.3", "output_tokens": 509, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:26:50.384743+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4d9ff84a-a9db-420b-8ff4-1ad86ee41465", "result_sha256": "23c63c6da5ef360e03f506d96d097f72981b34d792a69de213ca43c94316ef18", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004956, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 3.9466904510045424, "model_name": "muse-spark-1.3", "output_tokens": 422, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:12:44.972888+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "823847fa-3e47-41f9-8f46-ab867072108f", "result_sha256": "e148630b9e476ee439f529dfeaf8b681bcb715fdfccf34e3cea2260d016654b9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00517275, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 5.054415164981037, "model_name": "muse-spark-1.3", "output_tokens": 473, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:10:57.357854+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "815190e7-b2ac-45c0-82b5-9f41aed3f37f", "result_sha256": "312da785edd44568334fb39b3d6f62776e814ccc25c3fa7a0049df73ddc0e086", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00533, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 5.7758921650238335, "model_name": "muse-spark-1.3", "output_tokens": 510, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:10:40.763619+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "ecd88d9f-52ab-48c3-98a8-305b6105c486", "result_sha256": "133b8da7181b52e83709726d697ec464fa0f48a9b8bdbe4144e2bc28bb2d8ad8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006044, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 7.618747737957165, "model_name": "muse-spark-1.3", "output_tokens": 678, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:25:15.700331+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "d109a398-09d8-48f3-921e-ea212629b67c", "result_sha256": "066c76cb16ca6088b8cdd05339f652c4426e31d99f25e3057e30b9481fd28820", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004701, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 5.152085245004855, "model_name": "muse-spark-1.3", "output_tokens": 362, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:12:01.326369+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "a5e2e1db-2001-443e-8e19-91f44d006659", "result_sha256": "d65fef06492dfcf52d3716fbc6103e22363e79fda903158b9f02797df2a83253", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006792, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 8.40305144910235, "model_name": "muse-spark-1.3", "output_tokens": 854, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:50.685901+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "cabd05a6-95b3-4668-af31-28b28bb0e097", "result_sha256": "027396b2641d844ff89c25ce66f4b0e09a760a8f95d8fcaa31e194eeaa7585ad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0065965, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2530, "key_errors": null, "latency_sec": 10.055344258085825, "model_name": "muse-spark-1.3", "output_tokens": 808, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "openai", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-17T03:27:01.843812+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "0c44a1bd-fdf7-4553-a6d7-8cf68913d024", "result_sha256": "4b6e4d0e4b1c250290a4da52488a91b6b91f207d0da18f69d89c3caccec40213", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00438875, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 4.027216835995205, "model_name": "muse-spark-1.3", "output_tokens": 295, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:16:09.885912+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "14b8f42a-e68a-4c6c-9320-874d475a0474", "result_sha256": "ef405ad66db689eb5a60b70608bc1a54404d90580e26a0bd2e6220c0d35ffeec", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004495, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.0196947570657358, "model_name": "muse-spark-1.3", "output_tokens": 320, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:34.989761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "b74d2160-9a9b-46b0-ba29-d63be9e5808d", "result_sha256": "92119bf8f70611ea6c78394c99380a42037473f0a4147499e76bb84e0f711082", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00394675, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 2.247738915029913, "model_name": "muse-spark-1.3", "output_tokens": 191, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:15:52.214828+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "4b1756b6-8738-429d-bcf2-8183c7181fb6", "result_sha256": "87922daaf062cd4eb0848610d726b67301880dc312bf2631b3eaf4db25cbed42", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00441, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.1057423130841926, "model_name": "muse-spark-1.3", "output_tokens": 300, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:26:13.572136+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "b7fa8773-5ad0-46e4-a9fe-426ba5e775b2", "result_sha256": "78019ef05d7abf58d42a1b36191f083360927b069cab84661752b5bfaface228", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004291, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 2.7506123109487817, "model_name": "muse-spark-1.3", "output_tokens": 272, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:18:06.064053+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "9731079b-a508-44cf-85d8-d37efa8cc9b2", "result_sha256": "5856a71cb48fadae80479882fa21cfc8347614440edfdc2666a6793dbd6209dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.004325, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.0686949479859322, "model_name": "muse-spark-1.3", "output_tokens": 280, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "openai", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-17T03:14:53.818412+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "08e20687-b5a8-482b-862e-0f1ae7d310cb", "result_sha256": "b152176a3d2bb7a136be851fee09750c07882b0d889abef03e38f15478e21048", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00475, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 4.132129449979402, "model_name": "muse-spark-1.3", "output_tokens": 380, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:10:22.236471+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "48ffb3a3-b9ca-42f4-a0ca-b5d7e3ccb45c", "result_sha256": "8784bd657cf8d32fcca76a23e061039516c130a5e05afa15d91a01ab2876a0db", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00439725, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.2946902750991285, "model_name": "muse-spark-1.3", "output_tokens": 297, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "openai", "raw_prediction": "{\"choice\":\"A\"}", "recorded_at": "2026-09-17T03:10:09.413840+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3f19c47-0178-4601-aee0-422950ef3553", "result_sha256": "9761581c77333e4c88974eab4943f20e24ee829f7069bf649a4716f5d9efa6a2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0046395, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.891973796999082, "model_name": "muse-spark-1.3", "output_tokens": 354, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "openai", "raw_prediction": "{\"choice\":\"B\"}", "recorded_at": "2026-09-17T03:13:52.769139+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "39eceec4-c888-4e2d-994f-f0b173399627", "result_sha256": "9fab539071d9ad05fae6b56dc1ac692de12616b8f82e582ce6561c57fd683542", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00492, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 4.339462675037794, "model_name": "muse-spark-1.3", "output_tokens": 420, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:24:35.209797+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "bf335cee-1f8f-4f24-8439-fe32e15ce28c", "result_sha256": "5ae77138fac7637d7d2c5c321a040558cd090bb23ca928f6da73d15ac41974f2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00447375, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.596363215940073, "model_name": "muse-spark-1.3", "output_tokens": 315, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "openai", "raw_prediction": "{\"choice\":\"C\"}", "recorded_at": "2026-09-17T03:18:58.682921+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "muse-spark-1.3", "status": "final", "included_in_comparison": true, "source_attempt_id": "294d5b7a-e659-4fef-b497-bb5bf0d46a05", "result_sha256": "3b30cc5d3ed173ec98801ee92d35928d03d627789a6a24c160235c6f8357e3fa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00458, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2508, "key_errors": null, "latency_sec": 3.743313462007791, "model_name": "muse-spark-1.3", "output_tokens": 340, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "openai", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-17T03:09:46.884907+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } } ] }